2024-11-26 16:40:32,255 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-26 16:40:32,271 main DEBUG Took 0.013549 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 16:40:32,272 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 16:40:32,272 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 16:40:32,273 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 16:40:32,275 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 16:40:32,285 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 16:40:32,307 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 16:40:32,309 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 16:40:32,309 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 16:40:32,310 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 16:40:32,311 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 16:40:32,311 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 16:40:32,312 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 16:40:32,313 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 16:40:32,313 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 16:40:32,314 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 16:40:32,315 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 16:40:32,315 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 16:40:32,316 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 16:40:32,316 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 16:40:32,317 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 16:40:32,317 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 16:40:32,317 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 16:40:32,318 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 16:40:32,318 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 16:40:32,318 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 16:40:32,319 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 16:40:32,319 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 16:40:32,320 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 16:40:32,320 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 16:40:32,321 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 16:40:32,322 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 16:40:32,323 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 16:40:32,325 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 16:40:32,328 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 16:40:32,328 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 16:40:32,330 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 16:40:32,330 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 16:40:32,343 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 16:40:32,347 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 16:40:32,349 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 16:40:32,350 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 16:40:32,350 main DEBUG createAppenders(={Console}) 2024-11-26 16:40:32,351 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-26 16:40:32,352 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-26 16:40:32,352 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-26 16:40:32,353 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 16:40:32,353 main DEBUG OutputStream closed 2024-11-26 16:40:32,354 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 16:40:32,354 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 16:40:32,355 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-26 16:40:32,469 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 16:40:32,472 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 16:40:32,474 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 16:40:32,475 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 16:40:32,476 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 16:40:32,476 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 16:40:32,477 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 16:40:32,477 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 16:40:32,478 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 16:40:32,478 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 16:40:32,479 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 16:40:32,479 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 16:40:32,480 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 16:40:32,480 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 16:40:32,481 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 16:40:32,481 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 16:40:32,482 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 16:40:32,483 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 16:40:32,486 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 16:40:32,486 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-26 16:40:32,488 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 16:40:32,489 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-26T16:40:32,879 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed 2024-11-26 16:40:32,884 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 16:40:32,885 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T16:40:32,901 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-26T16:40:32,949 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T16:40:32,972 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/cluster_5c35ce1e-289c-85d9-824c-12c667b295d2, deleteOnExit=true 2024-11-26T16:40:32,973 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T16:40:32,974 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/test.cache.data in system properties and HBase conf 2024-11-26T16:40:32,975 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T16:40:32,976 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/hadoop.log.dir in system properties and HBase conf 2024-11-26T16:40:32,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T16:40:32,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T16:40:32,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T16:40:33,099 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T16:40:33,228 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T16:40:33,234 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T16:40:33,235 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T16:40:33,236 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T16:40:33,237 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T16:40:33,238 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T16:40:33,238 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T16:40:33,239 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T16:40:33,239 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T16:40:33,240 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T16:40:33,241 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/nfs.dump.dir in system properties and HBase conf 2024-11-26T16:40:33,241 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/java.io.tmpdir in system properties and HBase conf 2024-11-26T16:40:33,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T16:40:33,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T16:40:33,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T16:40:34,228 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-26T16:40:34,327 INFO [Time-limited test {}] log.Log(170): Logging initialized @2982ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-26T16:40:34,425 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T16:40:34,504 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T16:40:34,538 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T16:40:34,538 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T16:40:34,540 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T16:40:34,557 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T16:40:34,562 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/hadoop.log.dir/,AVAILABLE} 2024-11-26T16:40:34,563 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T16:40:34,805 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2341a969{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/java.io.tmpdir/jetty-localhost-45013-hadoop-hdfs-3_4_1-tests_jar-_-any-1555704882648047890/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T16:40:34,813 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:45013} 2024-11-26T16:40:34,813 INFO [Time-limited test {}] server.Server(415): Started @3470ms 2024-11-26T16:40:35,233 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T16:40:35,242 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T16:40:35,245 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T16:40:35,245 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T16:40:35,245 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T16:40:35,246 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/hadoop.log.dir/,AVAILABLE} 2024-11-26T16:40:35,247 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T16:40:35,383 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@292d50c4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/java.io.tmpdir/jetty-localhost-44665-hadoop-hdfs-3_4_1-tests_jar-_-any-12154805907160673025/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T16:40:35,384 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:44665} 2024-11-26T16:40:35,385 INFO [Time-limited test {}] server.Server(415): Started @4041ms 2024-11-26T16:40:35,442 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T16:40:36,030 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/cluster_5c35ce1e-289c-85d9-824c-12c667b295d2/data/data1/current/BP-338185642-172.17.0.2-1732639233957/current, will proceed with Du for space computation calculation, 2024-11-26T16:40:36,030 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/cluster_5c35ce1e-289c-85d9-824c-12c667b295d2/data/data2/current/BP-338185642-172.17.0.2-1732639233957/current, will proceed with Du for space computation calculation, 2024-11-26T16:40:36,105 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T16:40:36,198 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x44efedbeba27f15a with lease ID 0xb7d48d857fbec176: Processing first storage report for DS-1ef9c72b-93d8-463e-800d-f16bea6cff53 from datanode DatanodeRegistration(127.0.0.1:41979, datanodeUuid=9ade134c-bb6c-4bdc-9e79-03c00e4a8431, infoPort=46155, infoSecurePort=0, ipcPort=43585, storageInfo=lv=-57;cid=testClusterID;nsid=994765134;c=1732639233957) 2024-11-26T16:40:36,200 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x44efedbeba27f15a with lease ID 0xb7d48d857fbec176: from storage DS-1ef9c72b-93d8-463e-800d-f16bea6cff53 node DatanodeRegistration(127.0.0.1:41979, datanodeUuid=9ade134c-bb6c-4bdc-9e79-03c00e4a8431, infoPort=46155, infoSecurePort=0, ipcPort=43585, storageInfo=lv=-57;cid=testClusterID;nsid=994765134;c=1732639233957), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-26T16:40:36,204 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x44efedbeba27f15a with lease ID 0xb7d48d857fbec176: Processing first storage report for DS-93040e45-cf41-420c-922b-3e7cb3c290c3 from datanode DatanodeRegistration(127.0.0.1:41979, datanodeUuid=9ade134c-bb6c-4bdc-9e79-03c00e4a8431, infoPort=46155, infoSecurePort=0, ipcPort=43585, storageInfo=lv=-57;cid=testClusterID;nsid=994765134;c=1732639233957) 2024-11-26T16:40:36,205 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x44efedbeba27f15a with lease ID 0xb7d48d857fbec176: from storage DS-93040e45-cf41-420c-922b-3e7cb3c290c3 node DatanodeRegistration(127.0.0.1:41979, datanodeUuid=9ade134c-bb6c-4bdc-9e79-03c00e4a8431, infoPort=46155, infoSecurePort=0, ipcPort=43585, storageInfo=lv=-57;cid=testClusterID;nsid=994765134;c=1732639233957), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T16:40:36,290 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed 2024-11-26T16:40:36,428 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/cluster_5c35ce1e-289c-85d9-824c-12c667b295d2/zookeeper_0, clientPort=49596, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/cluster_5c35ce1e-289c-85d9-824c-12c667b295d2/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/cluster_5c35ce1e-289c-85d9-824c-12c667b295d2/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T16:40:36,444 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49596 2024-11-26T16:40:36,461 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T16:40:36,466 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T16:40:36,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741825_1001 (size=7) 2024-11-26T16:40:37,168 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711 with version=8 2024-11-26T16:40:37,169 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/hbase-staging 2024-11-26T16:40:37,268 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T16:40:37,532 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2dedfe7466e8:0 server-side Connection retries=45 2024-11-26T16:40:37,543 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T16:40:37,543 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T16:40:37,548 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T16:40:37,548 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T16:40:37,548 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T16:40:37,704 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T16:40:37,786 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T16:40:37,797 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T16:40:37,801 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T16:40:37,834 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1559 (auto-detected) 2024-11-26T16:40:37,836 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-26T16:40:37,861 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35233 2024-11-26T16:40:37,890 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35233 connecting to ZooKeeper ensemble=127.0.0.1:49596 2024-11-26T16:40:37,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:352330x0, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T16:40:37,932 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35233-0x10034b45a0d0000 connected 2024-11-26T16:40:37,965 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T16:40:37,969 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T16:40:37,981 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T16:40:37,985 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711, hbase.cluster.distributed=false 2024-11-26T16:40:38,014 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T16:40:38,026 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35233 2024-11-26T16:40:38,026 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35233 2024-11-26T16:40:38,027 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35233 2024-11-26T16:40:38,028 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35233 2024-11-26T16:40:38,028 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35233 2024-11-26T16:40:38,182 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2dedfe7466e8:0 server-side Connection retries=45 2024-11-26T16:40:38,185 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T16:40:38,185 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T16:40:38,185 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T16:40:38,186 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T16:40:38,186 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T16:40:38,189 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T16:40:38,191 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T16:40:38,192 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38109 2024-11-26T16:40:38,194 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38109 connecting to ZooKeeper ensemble=127.0.0.1:49596 2024-11-26T16:40:38,195 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T16:40:38,199 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T16:40:38,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:381090x0, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T16:40:38,210 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38109-0x10034b45a0d0001 connected 2024-11-26T16:40:38,210 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T16:40:38,218 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T16:40:38,229 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T16:40:38,232 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T16:40:38,238 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T16:40:38,239 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38109 2024-11-26T16:40:38,240 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38109 2024-11-26T16:40:38,241 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38109 2024-11-26T16:40:38,241 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38109 2024-11-26T16:40:38,242 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38109 2024-11-26T16:40:38,265 DEBUG [M:0;2dedfe7466e8:35233 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2dedfe7466e8:35233 2024-11-26T16:40:38,266 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2dedfe7466e8,35233,1732639237322 2024-11-26T16:40:38,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T16:40:38,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T16:40:38,277 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2dedfe7466e8,35233,1732639237322 2024-11-26T16:40:38,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T16:40:38,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T16:40:38,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T16:40:38,305 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T16:40:38,306 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2dedfe7466e8,35233,1732639237322 from backup master directory 2024-11-26T16:40:38,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T16:40:38,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2dedfe7466e8,35233,1732639237322 2024-11-26T16:40:38,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T16:40:38,311 WARN [master/2dedfe7466e8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T16:40:38,311 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2dedfe7466e8,35233,1732639237322 2024-11-26T16:40:38,314 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-26T16:40:38,315 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-26T16:40:38,396 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/hbase.id] with ID: 0d93ae86-e698-43e8-8f2c-76f242eed8e3 2024-11-26T16:40:38,397 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/.tmp/hbase.id 2024-11-26T16:40:38,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741826_1002 (size=42) 2024-11-26T16:40:38,812 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/.tmp/hbase.id]:[hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/hbase.id] 2024-11-26T16:40:38,864 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T16:40:38,870 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T16:40:38,891 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-26T16:40:38,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T16:40:38,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T16:40:38,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741827_1003 (size=196) 2024-11-26T16:40:38,932 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T16:40:38,935 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T16:40:38,954 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:38,959 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T16:40:38,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741828_1004 (size=1189) 2024-11-26T16:40:39,415 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/data/master/store 2024-11-26T16:40:39,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741829_1005 (size=34) 2024-11-26T16:40:39,439 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T16:40:39,442 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T16:40:39,443 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T16:40:39,444 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T16:40:39,444 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T16:40:39,445 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T16:40:39,445 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T16:40:39,445 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T16:40:39,447 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732639239443Disabling compacts and flushes for region at 1732639239443Disabling writes for close at 1732639239445 (+2 ms)Writing region close event to WAL at 1732639239445Closed at 1732639239445 2024-11-26T16:40:39,449 WARN [master/2dedfe7466e8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/data/master/store/.initializing 2024-11-26T16:40:39,449 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/WALs/2dedfe7466e8,35233,1732639237322 2024-11-26T16:40:39,457 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T16:40:39,473 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2dedfe7466e8%2C35233%2C1732639237322, suffix=, logDir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/WALs/2dedfe7466e8,35233,1732639237322, archiveDir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/oldWALs, maxLogs=10 2024-11-26T16:40:39,505 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/WALs/2dedfe7466e8,35233,1732639237322/2dedfe7466e8%2C35233%2C1732639237322.1732639239479, exclude list is [], retry=0 2024-11-26T16:40:39,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41979,DS-1ef9c72b-93d8-463e-800d-f16bea6cff53,DISK] 2024-11-26T16:40:39,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-26T16:40:39,569 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/WALs/2dedfe7466e8,35233,1732639237322/2dedfe7466e8%2C35233%2C1732639237322.1732639239479 2024-11-26T16:40:39,570 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46155:46155)] 2024-11-26T16:40:39,571 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T16:40:39,571 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T16:40:39,575 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T16:40:39,576 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T16:40:39,622 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T16:40:39,654 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T16:40:39,659 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:40:39,663 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T16:40:39,664 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T16:40:39,668 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T16:40:39,668 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:40:39,670 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:40:39,670 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T16:40:39,674 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T16:40:39,674 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:40:39,675 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:40:39,676 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T16:40:39,679 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T16:40:39,680 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:40:39,681 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:40:39,681 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T16:40:39,686 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T16:40:39,687 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T16:40:39,693 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T16:40:39,694 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T16:40:39,698 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T16:40:39,702 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T16:40:39,707 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T16:40:39,708 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67430803, jitterRate=0.004797264933586121}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T16:40:39,713 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732639239591Initializing all the Stores at 1732639239594 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732639239594Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639239595 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639239595Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639239595Cleaning up temporary data from old regions at 1732639239694 (+99 ms)Region opened successfully at 1732639239713 (+19 ms) 2024-11-26T16:40:39,715 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T16:40:39,760 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b8984a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2dedfe7466e8/172.17.0.2:0 2024-11-26T16:40:39,805 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T16:40:39,818 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T16:40:39,818 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T16:40:39,821 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T16:40:39,823 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T16:40:39,829 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-26T16:40:39,829 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T16:40:39,861 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T16:40:39,872 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T16:40:39,875 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T16:40:39,879 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T16:40:39,881 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T16:40:39,882 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T16:40:39,885 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T16:40:39,889 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T16:40:39,891 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T16:40:39,892 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T16:40:39,894 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T16:40:39,915 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T16:40:39,917 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T16:40:39,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T16:40:39,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T16:40:39,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T16:40:39,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T16:40:39,925 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2dedfe7466e8,35233,1732639237322, sessionid=0x10034b45a0d0000, setting cluster-up flag (Was=false) 2024-11-26T16:40:39,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T16:40:39,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T16:40:39,950 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T16:40:39,952 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2dedfe7466e8,35233,1732639237322 2024-11-26T16:40:39,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T16:40:39,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T16:40:39,964 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T16:40:39,966 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2dedfe7466e8,35233,1732639237322 2024-11-26T16:40:39,973 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T16:40:40,049 INFO [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(746): ClusterId : 0d93ae86-e698-43e8-8f2c-76f242eed8e3 2024-11-26T16:40:40,052 DEBUG [RS:0;2dedfe7466e8:38109 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T16:40:40,059 DEBUG [RS:0;2dedfe7466e8:38109 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T16:40:40,059 DEBUG [RS:0;2dedfe7466e8:38109 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T16:40:40,063 DEBUG [RS:0;2dedfe7466e8:38109 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T16:40:40,063 DEBUG [RS:0;2dedfe7466e8:38109 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@347aeef2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2dedfe7466e8/172.17.0.2:0 2024-11-26T16:40:40,063 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T16:40:40,076 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T16:40:40,085 DEBUG [RS:0;2dedfe7466e8:38109 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2dedfe7466e8:38109 2024-11-26T16:40:40,087 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T16:40:40,090 INFO [RS:0;2dedfe7466e8:38109 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T16:40:40,090 INFO [RS:0;2dedfe7466e8:38109 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T16:40:40,090 DEBUG [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T16:40:40,094 INFO [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(2659): reportForDuty to master=2dedfe7466e8,35233,1732639237322 with port=38109, startcode=1732639238134 2024-11-26T16:40:40,096 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2dedfe7466e8,35233,1732639237322 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T16:40:40,105 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2dedfe7466e8:0, corePoolSize=5, maxPoolSize=5 2024-11-26T16:40:40,105 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2dedfe7466e8:0, corePoolSize=5, maxPoolSize=5 2024-11-26T16:40:40,105 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2dedfe7466e8:0, corePoolSize=5, maxPoolSize=5 2024-11-26T16:40:40,105 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2dedfe7466e8:0, corePoolSize=5, maxPoolSize=5 2024-11-26T16:40:40,105 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2dedfe7466e8:0, corePoolSize=10, maxPoolSize=10 2024-11-26T16:40:40,106 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2dedfe7466e8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T16:40:40,106 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2dedfe7466e8:0, corePoolSize=2, maxPoolSize=2 2024-11-26T16:40:40,106 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2dedfe7466e8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T16:40:40,109 DEBUG [RS:0;2dedfe7466e8:38109 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T16:40:40,113 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732639270112 2024-11-26T16:40:40,114 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T16:40:40,115 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T16:40:40,118 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T16:40:40,119 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T16:40:40,119 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T16:40:40,119 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T16:40:40,119 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T16:40:40,120 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T16:40:40,120 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:40,125 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T16:40:40,126 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T16:40:40,126 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T16:40:40,127 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:40:40,127 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T16:40:40,128 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T16:40:40,129 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T16:40:40,131 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2dedfe7466e8:0:becomeActiveMaster-HFileCleaner.large.0-1732639240130,5,FailOnTimeoutGroup] 2024-11-26T16:40:40,132 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2dedfe7466e8:0:becomeActiveMaster-HFileCleaner.small.0-1732639240131,5,FailOnTimeoutGroup] 2024-11-26T16:40:40,132 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:40,132 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T16:40:40,134 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:40,134 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:40,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741831_1007 (size=1321) 2024-11-26T16:40:40,149 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54677, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T16:40:40,156 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35233 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:40,158 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35233 {}] master.ServerManager(517): Registering regionserver=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:40,176 DEBUG [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711 2024-11-26T16:40:40,176 DEBUG [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44115 2024-11-26T16:40:40,176 DEBUG [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T16:40:40,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T16:40:40,181 DEBUG [RS:0;2dedfe7466e8:38109 {}] zookeeper.ZKUtil(111): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:40,181 WARN [RS:0;2dedfe7466e8:38109 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T16:40:40,182 INFO [RS:0;2dedfe7466e8:38109 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T16:40:40,182 DEBUG [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/WALs/2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:40,184 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2dedfe7466e8,38109,1732639238134] 2024-11-26T16:40:40,210 INFO [RS:0;2dedfe7466e8:38109 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T16:40:40,224 INFO [RS:0;2dedfe7466e8:38109 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T16:40:40,230 INFO [RS:0;2dedfe7466e8:38109 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T16:40:40,230 INFO [RS:0;2dedfe7466e8:38109 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:40,231 INFO [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T16:40:40,237 INFO [RS:0;2dedfe7466e8:38109 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T16:40:40,239 INFO [RS:0;2dedfe7466e8:38109 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:40,239 DEBUG [RS:0;2dedfe7466e8:38109 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2dedfe7466e8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T16:40:40,239 DEBUG [RS:0;2dedfe7466e8:38109 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2dedfe7466e8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T16:40:40,239 DEBUG [RS:0;2dedfe7466e8:38109 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2dedfe7466e8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T16:40:40,239 DEBUG [RS:0;2dedfe7466e8:38109 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2dedfe7466e8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T16:40:40,240 DEBUG [RS:0;2dedfe7466e8:38109 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2dedfe7466e8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T16:40:40,240 DEBUG [RS:0;2dedfe7466e8:38109 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2dedfe7466e8:0, corePoolSize=2, maxPoolSize=2 2024-11-26T16:40:40,240 DEBUG [RS:0;2dedfe7466e8:38109 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2dedfe7466e8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T16:40:40,240 DEBUG [RS:0;2dedfe7466e8:38109 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2dedfe7466e8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T16:40:40,240 DEBUG [RS:0;2dedfe7466e8:38109 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2dedfe7466e8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T16:40:40,241 DEBUG [RS:0;2dedfe7466e8:38109 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2dedfe7466e8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T16:40:40,241 DEBUG [RS:0;2dedfe7466e8:38109 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2dedfe7466e8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T16:40:40,241 DEBUG [RS:0;2dedfe7466e8:38109 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2dedfe7466e8:0, corePoolSize=1, maxPoolSize=1 2024-11-26T16:40:40,241 DEBUG [RS:0;2dedfe7466e8:38109 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2dedfe7466e8:0, corePoolSize=3, maxPoolSize=3 2024-11-26T16:40:40,241 DEBUG [RS:0;2dedfe7466e8:38109 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0, corePoolSize=3, maxPoolSize=3 2024-11-26T16:40:40,242 INFO [RS:0;2dedfe7466e8:38109 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:40,242 INFO [RS:0;2dedfe7466e8:38109 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:40,243 INFO [RS:0;2dedfe7466e8:38109 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:40,243 INFO [RS:0;2dedfe7466e8:38109 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:40,243 INFO [RS:0;2dedfe7466e8:38109 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:40,244 INFO [RS:0;2dedfe7466e8:38109 {}] hbase.ChoreService(168): Chore ScheduledChore name=2dedfe7466e8,38109,1732639238134-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T16:40:40,264 INFO [RS:0;2dedfe7466e8:38109 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T16:40:40,266 INFO [RS:0;2dedfe7466e8:38109 {}] hbase.ChoreService(168): Chore ScheduledChore name=2dedfe7466e8,38109,1732639238134-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:40,267 INFO [RS:0;2dedfe7466e8:38109 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:40,267 INFO [RS:0;2dedfe7466e8:38109 {}] regionserver.Replication(171): 2dedfe7466e8,38109,1732639238134 started 2024-11-26T16:40:40,286 INFO [RS:0;2dedfe7466e8:38109 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:40,287 INFO [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(1482): Serving as 2dedfe7466e8,38109,1732639238134, RpcServer on 2dedfe7466e8/172.17.0.2:38109, sessionid=0x10034b45a0d0001 2024-11-26T16:40:40,288 DEBUG [RS:0;2dedfe7466e8:38109 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T16:40:40,288 DEBUG [RS:0;2dedfe7466e8:38109 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:40,288 DEBUG [RS:0;2dedfe7466e8:38109 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2dedfe7466e8,38109,1732639238134' 2024-11-26T16:40:40,288 DEBUG [RS:0;2dedfe7466e8:38109 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T16:40:40,289 DEBUG [RS:0;2dedfe7466e8:38109 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T16:40:40,290 DEBUG [RS:0;2dedfe7466e8:38109 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T16:40:40,290 DEBUG [RS:0;2dedfe7466e8:38109 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T16:40:40,290 DEBUG [RS:0;2dedfe7466e8:38109 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:40,291 DEBUG [RS:0;2dedfe7466e8:38109 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2dedfe7466e8,38109,1732639238134' 2024-11-26T16:40:40,291 DEBUG [RS:0;2dedfe7466e8:38109 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T16:40:40,291 DEBUG [RS:0;2dedfe7466e8:38109 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T16:40:40,292 DEBUG [RS:0;2dedfe7466e8:38109 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T16:40:40,292 INFO [RS:0;2dedfe7466e8:38109 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T16:40:40,292 INFO [RS:0;2dedfe7466e8:38109 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T16:40:40,398 INFO [RS:0;2dedfe7466e8:38109 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T16:40:40,401 INFO [RS:0;2dedfe7466e8:38109 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2dedfe7466e8%2C38109%2C1732639238134, suffix=, logDir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/WALs/2dedfe7466e8,38109,1732639238134, archiveDir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/oldWALs, maxLogs=32 2024-11-26T16:40:40,418 DEBUG [RS:0;2dedfe7466e8:38109 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/WALs/2dedfe7466e8,38109,1732639238134/2dedfe7466e8%2C38109%2C1732639238134.1732639240403, exclude list is [], retry=0 2024-11-26T16:40:40,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41979,DS-1ef9c72b-93d8-463e-800d-f16bea6cff53,DISK] 2024-11-26T16:40:40,426 INFO [RS:0;2dedfe7466e8:38109 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/WALs/2dedfe7466e8,38109,1732639238134/2dedfe7466e8%2C38109%2C1732639238134.1732639240403 2024-11-26T16:40:40,426 DEBUG [RS:0;2dedfe7466e8:38109 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46155:46155)] 2024-11-26T16:40:40,541 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T16:40:40,542 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711 2024-11-26T16:40:40,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741833_1009 (size=32) 2024-11-26T16:40:40,954 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T16:40:40,957 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T16:40:40,960 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T16:40:40,960 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:40:40,961 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T16:40:40,961 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T16:40:40,964 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T16:40:40,964 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:40:40,965 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T16:40:40,965 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T16:40:40,967 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T16:40:40,968 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:40:40,968 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T16:40:40,969 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T16:40:40,971 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T16:40:40,971 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:40:40,972 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T16:40:40,972 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T16:40:40,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740 2024-11-26T16:40:40,974 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740 2024-11-26T16:40:40,977 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T16:40:40,977 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T16:40:40,978 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T16:40:40,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T16:40:40,984 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T16:40:40,985 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62271635, jitterRate=-0.0720803290605545}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T16:40:40,987 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732639240954Initializing all the Stores at 1732639240956 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732639240956Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732639240957 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639240957Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732639240957Cleaning up temporary data from old regions at 1732639240977 (+20 ms)Region opened successfully at 1732639240987 (+10 ms) 2024-11-26T16:40:40,988 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T16:40:40,988 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T16:40:40,988 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T16:40:40,988 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T16:40:40,988 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T16:40:40,989 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T16:40:40,989 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732639240988Disabling compacts and flushes for region at 1732639240988Disabling writes for close at 1732639240988Writing region close event to WAL at 1732639240989 (+1 ms)Closed at 1732639240989 2024-11-26T16:40:40,993 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T16:40:40,993 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T16:40:41,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T16:40:41,008 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T16:40:41,011 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T16:40:41,164 DEBUG [2dedfe7466e8:35233 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-26T16:40:41,176 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:41,184 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2dedfe7466e8,38109,1732639238134, state=OPENING 2024-11-26T16:40:41,189 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T16:40:41,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T16:40:41,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T16:40:41,191 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T16:40:41,192 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T16:40:41,193 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T16:40:41,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2dedfe7466e8,38109,1732639238134}] 2024-11-26T16:40:41,373 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T16:40:41,377 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53645, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T16:40:41,388 INFO [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T16:40:41,389 INFO [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T16:40:41,389 INFO [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T16:40:41,393 INFO [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2dedfe7466e8%2C38109%2C1732639238134.meta, suffix=.meta, logDir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/WALs/2dedfe7466e8,38109,1732639238134, archiveDir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/oldWALs, maxLogs=32 2024-11-26T16:40:41,409 DEBUG [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/WALs/2dedfe7466e8,38109,1732639238134/2dedfe7466e8%2C38109%2C1732639238134.meta.1732639241394.meta, exclude list is [], retry=0 2024-11-26T16:40:41,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41979,DS-1ef9c72b-93d8-463e-800d-f16bea6cff53,DISK] 2024-11-26T16:40:41,416 INFO [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/WALs/2dedfe7466e8,38109,1732639238134/2dedfe7466e8%2C38109%2C1732639238134.meta.1732639241394.meta 2024-11-26T16:40:41,417 DEBUG [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46155:46155)] 2024-11-26T16:40:41,417 DEBUG [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T16:40:41,419 DEBUG [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T16:40:41,422 DEBUG [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T16:40:41,428 INFO [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T16:40:41,432 DEBUG [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T16:40:41,433 DEBUG [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T16:40:41,433 DEBUG [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T16:40:41,433 DEBUG [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T16:40:41,437 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T16:40:41,438 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T16:40:41,438 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:40:41,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T16:40:41,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T16:40:41,441 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T16:40:41,441 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:40:41,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T16:40:41,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T16:40:41,443 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T16:40:41,443 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:40:41,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T16:40:41,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T16:40:41,446 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T16:40:41,446 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:40:41,446 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T16:40:41,447 DEBUG [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T16:40:41,448 DEBUG [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740 2024-11-26T16:40:41,450 DEBUG [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740 2024-11-26T16:40:41,452 DEBUG [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T16:40:41,453 DEBUG [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T16:40:41,453 DEBUG [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T16:40:41,456 DEBUG [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T16:40:41,457 INFO [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59746543, jitterRate=-0.10970713198184967}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T16:40:41,457 DEBUG [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T16:40:41,459 DEBUG [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732639241434Writing region info on filesystem at 1732639241434Initializing all the Stores at 1732639241436 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732639241436Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732639241436Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639241437 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732639241437Cleaning up temporary data from old regions at 1732639241453 (+16 ms)Running coprocessor post-open hooks at 1732639241457 (+4 ms)Region opened successfully at 1732639241459 (+2 ms) 2024-11-26T16:40:41,466 INFO [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732639241363 2024-11-26T16:40:41,478 DEBUG [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T16:40:41,478 INFO [RS_OPEN_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T16:40:41,480 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:41,482 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2dedfe7466e8,38109,1732639238134, state=OPEN 2024-11-26T16:40:41,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T16:40:41,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T16:40:41,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T16:40:41,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T16:40:41,488 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:41,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T16:40:41,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2dedfe7466e8,38109,1732639238134 in 293 msec 2024-11-26T16:40:41,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T16:40:41,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 496 msec 2024-11-26T16:40:41,501 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T16:40:41,501 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T16:40:41,524 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:40:41,525 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:40:41,548 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:40:41,551 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53063, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:40:41,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5640 sec 2024-11-26T16:40:41,573 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732639241572, completionTime=-1 2024-11-26T16:40:41,576 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-26T16:40:41,577 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T16:40:41,630 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-26T16:40:41,631 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732639301631 2024-11-26T16:40:41,631 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732639361631 2024-11-26T16:40:41,631 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 54 msec 2024-11-26T16:40:41,633 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2dedfe7466e8,35233,1732639237322-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:41,633 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2dedfe7466e8,35233,1732639237322-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:41,634 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2dedfe7466e8,35233,1732639237322-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:41,638 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2dedfe7466e8:35233, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:41,639 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:41,639 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:41,646 DEBUG [master/2dedfe7466e8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T16:40:41,666 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.355sec 2024-11-26T16:40:41,667 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T16:40:41,669 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T16:40:41,670 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T16:40:41,670 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T16:40:41,670 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T16:40:41,671 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2dedfe7466e8,35233,1732639237322-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T16:40:41,672 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2dedfe7466e8,35233,1732639237322-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T16:40:41,679 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T16:40:41,680 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T16:40:41,680 INFO [master/2dedfe7466e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2dedfe7466e8,35233,1732639237322-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T16:40:41,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c90ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:41,760 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T16:40:41,760 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T16:40:41,763 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:40:41,766 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:40:41,774 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:40:41,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:40:41,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:40:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ecb092, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:40:41,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:40:41,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:40:41,785 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:40:41,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ba7d24a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:41,788 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:40:41,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:40:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:40:41,797 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:40:41,818 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2dedfe7466e8,35233,1732639237322 2024-11-26T16:40:41,834 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=275, ProcessCount=11, AvailableMemoryMB=6520 2024-11-26T16:40:41,842 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T16:40:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 2dedfe7466e8,35233,1732639237322 2024-11-26T16:40:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@660d4cc9 2024-11-26T16:40:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T16:40:41,853 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T16:40:41,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T16:40:41,860 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T16:40:41,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T16:40:41,871 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T16:40:41,873 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-26T16:40:41,873 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:40:41,876 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T16:40:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T16:40:41,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741835_1011 (size=1061) 2024-11-26T16:40:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T16:40:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T16:40:42,291 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711 2024-11-26T16:40:42,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741836_1012 (size=53) 2024-11-26T16:40:42,299 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T16:40:42,300 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7c6dff940b7962766483b28d1008c073, disabling compactions & flushes 2024-11-26T16:40:42,300 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:42,300 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:42,300 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. after waiting 0 ms 2024-11-26T16:40:42,300 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:42,300 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:42,300 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7c6dff940b7962766483b28d1008c073: Waiting for close lock at 1732639242300Disabling compacts and flushes for region at 1732639242300Disabling writes for close at 1732639242300Writing region close event to WAL at 1732639242300Closed at 1732639242300 2024-11-26T16:40:42,302 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T16:40:42,307 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732639242302"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732639242302"}]},"ts":"1732639242302"} 2024-11-26T16:40:42,312 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T16:40:42,314 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T16:40:42,317 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639242314"}]},"ts":"1732639242314"} 2024-11-26T16:40:42,321 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T16:40:42,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c6dff940b7962766483b28d1008c073, ASSIGN}] 2024-11-26T16:40:42,325 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c6dff940b7962766483b28d1008c073, ASSIGN 2024-11-26T16:40:42,327 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c6dff940b7962766483b28d1008c073, ASSIGN; state=OFFLINE, location=2dedfe7466e8,38109,1732639238134; forceNewPlan=false, retain=false 2024-11-26T16:40:42,478 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7c6dff940b7962766483b28d1008c073, regionState=OPENING, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:42,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c6dff940b7962766483b28d1008c073, ASSIGN because future has completed 2024-11-26T16:40:42,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134}] 2024-11-26T16:40:42,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T16:40:42,646 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:42,646 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} 2024-11-26T16:40:42,647 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:42,647 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T16:40:42,647 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:42,647 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:42,650 INFO [StoreOpener-7c6dff940b7962766483b28d1008c073-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:42,654 INFO [StoreOpener-7c6dff940b7962766483b28d1008c073-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:40:42,655 INFO [StoreOpener-7c6dff940b7962766483b28d1008c073-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c6dff940b7962766483b28d1008c073 columnFamilyName A 2024-11-26T16:40:42,655 DEBUG [StoreOpener-7c6dff940b7962766483b28d1008c073-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:40:42,656 INFO [StoreOpener-7c6dff940b7962766483b28d1008c073-1 {}] regionserver.HStore(327): Store=7c6dff940b7962766483b28d1008c073/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:40:42,657 INFO [StoreOpener-7c6dff940b7962766483b28d1008c073-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:42,659 INFO [StoreOpener-7c6dff940b7962766483b28d1008c073-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:40:42,659 INFO [StoreOpener-7c6dff940b7962766483b28d1008c073-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c6dff940b7962766483b28d1008c073 columnFamilyName B 2024-11-26T16:40:42,659 DEBUG [StoreOpener-7c6dff940b7962766483b28d1008c073-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:40:42,660 INFO [StoreOpener-7c6dff940b7962766483b28d1008c073-1 {}] regionserver.HStore(327): Store=7c6dff940b7962766483b28d1008c073/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:40:42,661 INFO [StoreOpener-7c6dff940b7962766483b28d1008c073-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:42,663 INFO [StoreOpener-7c6dff940b7962766483b28d1008c073-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:40:42,663 INFO [StoreOpener-7c6dff940b7962766483b28d1008c073-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c6dff940b7962766483b28d1008c073 columnFamilyName C 2024-11-26T16:40:42,664 DEBUG [StoreOpener-7c6dff940b7962766483b28d1008c073-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:40:42,664 INFO [StoreOpener-7c6dff940b7962766483b28d1008c073-1 {}] regionserver.HStore(327): Store=7c6dff940b7962766483b28d1008c073/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:40:42,665 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:42,665 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:42,666 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:42,667 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:42,669 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:42,669 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:42,670 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T16:40:42,672 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:42,675 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T16:40:42,676 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7c6dff940b7962766483b28d1008c073; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60153153, jitterRate=-0.10364817082881927}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T16:40:42,676 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:42,677 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7c6dff940b7962766483b28d1008c073: Running coprocessor pre-open hook at 1732639242647Writing region info on filesystem at 1732639242647Initializing all the Stores at 1732639242649 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639242649Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639242649Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639242650 (+1 ms)Cleaning up temporary data from old regions at 1732639242669 (+19 ms)Running coprocessor post-open hooks at 1732639242676 (+7 ms)Region opened successfully at 1732639242677 (+1 ms) 2024-11-26T16:40:42,679 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., pid=6, masterSystemTime=1732639242638 2024-11-26T16:40:42,683 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:42,683 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:42,684 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7c6dff940b7962766483b28d1008c073, regionState=OPEN, openSeqNum=2, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:42,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 because future has completed 2024-11-26T16:40:42,694 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T16:40:42,694 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 in 206 msec 2024-11-26T16:40:42,698 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T16:40:42,698 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c6dff940b7962766483b28d1008c073, ASSIGN in 371 msec 2024-11-26T16:40:42,700 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T16:40:42,700 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639242700"}]},"ts":"1732639242700"} 2024-11-26T16:40:42,703 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T16:40:42,705 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T16:40:42,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 842 msec 2024-11-26T16:40:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T16:40:43,023 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:40:43,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c8b07f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,029 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:40:43,029 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:40:43,030 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:40:43,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:40:43,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:40:43,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24219159, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:40:43,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:40:43,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:40:43,034 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:40:43,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b3feb14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,037 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:40:43,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:40:43,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:40:43,041 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:40:43,043 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T16:40:43,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 2dedfe7466e8,35233,1732639237322 2024-11-26T16:40:43,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6f451fdf 2024-11-26T16:40:43,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T16:40:43,050 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T16:40:43,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff18d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,055 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:40:43,055 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:40:43,057 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:40:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:40:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:40:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b3d2fd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:40:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:40:43,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:40:43,060 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:40:43,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c860641, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109bd06b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,062 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:40:43,062 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:40:43,063 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:40:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:40:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:40:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@787f40de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:40:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:40:43,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:40:43,066 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:40:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5676fb0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,070 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af20e9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,071 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:40:43,071 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:40:43,072 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:40:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:40:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:40:43,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2544cc55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:40:43,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:40:43,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:40:43,077 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:40:43,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63181207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7387a89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,079 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:40:43,079 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:40:43,081 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:40:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:40:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:40:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d22a118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:40:43,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:40:43,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:40:43,084 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:40:43,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8e0b13b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75a031a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,086 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:40:43,086 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:40:43,087 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:40:43,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:40:43,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:40:43,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@831686b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:40:43,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:40:43,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:40:43,089 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:40:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14954dd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@276a94e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,091 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:40:43,091 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:40:43,092 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:40:43,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:40:43,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:40:43,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11dd0198, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:40:43,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:40:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:40:43,095 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:40:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cff1959, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d403639, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:40:43,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:40:43,098 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:40:43,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:40:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:40:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7788dec7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:40:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:40:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:40:43,100 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:40:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20ec50e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52d66d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,102 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:40:43,103 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:40:43,103 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:40:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:40:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:40:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dc64f57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:40:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:40:43,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:40:43,105 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:40:43,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3adce01e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,107 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7be0d766, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,107 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:40:43,107 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:40:43,108 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:40:43,108 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:40:43,108 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:40:43,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a0271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:40:43,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:40:43,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:40:43,110 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:40:43,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9402b31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:40:43,115 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:40:43,115 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:40:43,115 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:40:43,115 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:40:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:40:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:40:43,118 DEBUG [Thread-149 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:40:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:40:43,118 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:40:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:40:43,118 DEBUG [Thread-149 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:40:43,119 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:40:43,119 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:40:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:40:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:40:43,120 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:40:43,120 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:40:43,120 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:40:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:40:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:40:43,121 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:40:43,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:40:43,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:40:43,122 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:40:43,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:40:43,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:40:43,123 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:40:43,124 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:40:43,124 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:40:43,125 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:40:43,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:40:43,125 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:40:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:40:43,127 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:40:43,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:40:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:40:43,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:40:43,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:40:43,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:40:43,131 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:40:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:40:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:40:43,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:40:43,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:40:43,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:40:43,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:40:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-26T16:40:43,146 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:40:43,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T16:40:43,153 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:40:43,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:40:43,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T16:40:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:43,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:40:43,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:43,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:40:43,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:43,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:40:43,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T16:40:43,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:43,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52792 deadline: 1732639303315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:43,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/57e15692b95d45f2b3c24dbdbeae9473 is 50, key is test_row_0/A:col10/1732639243199/Put/seqid=0 2024-11-26T16:40:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52758 deadline: 1732639303312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:43,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:43,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:43,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52818 deadline: 1732639303323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:43,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T16:40:43,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639303322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:43,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:43,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639303333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:43,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:43,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:43,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:43,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:43,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:43,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:43,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:43,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:43,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:43,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:43,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741837_1013 (size=12001) 2024-11-26T16:40:43,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/57e15692b95d45f2b3c24dbdbeae9473 2024-11-26T16:40:43,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T16:40:43,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52792 deadline: 1732639303482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:43,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52818 deadline: 1732639303484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:43,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:43,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639303485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:43,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:43,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:43,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52758 deadline: 1732639303485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:43,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639303485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:43,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:43,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/281066a74a13447ab789ca5d9e973913 is 50, key is test_row_0/B:col10/1732639243199/Put/seqid=0 2024-11-26T16:40:43,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T16:40:43,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:43,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:43,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:43,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:43,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:43,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741838_1014 (size=12001) 2024-11-26T16:40:43,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/281066a74a13447ab789ca5d9e973913 2024-11-26T16:40:43,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/49b91840b6ef40ea802b12272e8a2c4c is 50, key is test_row_0/C:col10/1732639243199/Put/seqid=0 2024-11-26T16:40:43,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741839_1015 (size=12001) 2024-11-26T16:40:43,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/49b91840b6ef40ea802b12272e8a2c4c 2024-11-26T16:40:43,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/57e15692b95d45f2b3c24dbdbeae9473 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/57e15692b95d45f2b3c24dbdbeae9473 2024-11-26T16:40:43,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/57e15692b95d45f2b3c24dbdbeae9473, entries=150, sequenceid=13, filesize=11.7 K 2024-11-26T16:40:43,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/281066a74a13447ab789ca5d9e973913 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/281066a74a13447ab789ca5d9e973913 2024-11-26T16:40:43,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T16:40:43,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:43,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:43,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:43,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:43,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:43,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:43,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/281066a74a13447ab789ca5d9e973913, entries=150, sequenceid=13, filesize=11.7 K 2024-11-26T16:40:43,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/49b91840b6ef40ea802b12272e8a2c4c as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/49b91840b6ef40ea802b12272e8a2c4c 2024-11-26T16:40:43,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/49b91840b6ef40ea802b12272e8a2c4c, entries=150, sequenceid=13, filesize=11.7 K 2024-11-26T16:40:43,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 7c6dff940b7962766483b28d1008c073 in 508ms, sequenceid=13, compaction requested=false 2024-11-26T16:40:43,726 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T16:40:43,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:43,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-26T16:40:43,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:40:43,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:40:43,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:40:43,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:40:43,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:40:43,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:40:43,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/d76ff77803b34044acd81258000e957a is 50, key is test_row_0/A:col10/1732639243323/Put/seqid=0 2024-11-26T16:40:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T16:40:43,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741840_1016 (size=16681) 2024-11-26T16:40:43,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:43,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639303767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52792 deadline: 1732639303771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:43,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:43,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:43,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52758 deadline: 1732639303771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639303775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:43,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:43,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52818 deadline: 1732639303778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:43,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:43,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T16:40:43,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:43,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:43,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:43,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:43,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:43,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639303905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:43,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52758 deadline: 1732639303906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:43,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52792 deadline: 1732639303905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:43,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:43,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52818 deadline: 1732639303907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:43,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639303910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:43,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:43,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T16:40:44,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:44,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:44,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:44,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52758 deadline: 1732639304112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:44,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52792 deadline: 1732639304122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52818 deadline: 1732639304123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:44,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639304124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:44,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639304124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T16:40:44,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:44,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:44,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:44,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/d76ff77803b34044acd81258000e957a 2024-11-26T16:40:44,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/3ee7d070623e4f6b9245c08b4d23f881 is 50, key is test_row_0/B:col10/1732639243323/Put/seqid=0 2024-11-26T16:40:44,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741841_1017 (size=12001) 2024-11-26T16:40:44,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/3ee7d070623e4f6b9245c08b4d23f881 2024-11-26T16:40:44,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/4f3d52925f2a40f78c7261c403708cb3 is 50, key is test_row_0/C:col10/1732639243323/Put/seqid=0 2024-11-26T16:40:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T16:40:44,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741842_1018 (size=12001) 2024-11-26T16:40:44,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/4f3d52925f2a40f78c7261c403708cb3 2024-11-26T16:40:44,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/d76ff77803b34044acd81258000e957a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/d76ff77803b34044acd81258000e957a 2024-11-26T16:40:44,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T16:40:44,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:44,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:44,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:44,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/d76ff77803b34044acd81258000e957a, entries=250, sequenceid=35, filesize=16.3 K 2024-11-26T16:40:44,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/3ee7d070623e4f6b9245c08b4d23f881 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/3ee7d070623e4f6b9245c08b4d23f881 2024-11-26T16:40:44,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/3ee7d070623e4f6b9245c08b4d23f881, entries=150, sequenceid=35, filesize=11.7 K 2024-11-26T16:40:44,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/4f3d52925f2a40f78c7261c403708cb3 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/4f3d52925f2a40f78c7261c403708cb3 2024-11-26T16:40:44,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/4f3d52925f2a40f78c7261c403708cb3, entries=150, sequenceid=35, filesize=11.7 K 2024-11-26T16:40:44,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for 7c6dff940b7962766483b28d1008c073 in 637ms, sequenceid=35, compaction requested=false 2024-11-26T16:40:44,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:44,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T16:40:44,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:40:44,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:44,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:40:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:44,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:44,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:40:44,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:44,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/611dc92837194c13ba0765006fa23697 is 50, key is test_row_0/A:col10/1732639243764/Put/seqid=0 2024-11-26T16:40:44,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741843_1019 (size=14341) 2024-11-26T16:40:44,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/611dc92837194c13ba0765006fa23697 2024-11-26T16:40:44,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T16:40:44,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:44,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:44,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:44,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/7377c20524ce4bb78991200b10234ff1 is 50, key is test_row_0/B:col10/1732639243764/Put/seqid=0 2024-11-26T16:40:44,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639304498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:44,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639304502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:44,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741844_1020 (size=12001) 2024-11-26T16:40:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639244490, max timestamp=1732639244513Num uniques 859; 2024-11-26T16:40:44,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:40:44,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639244490, max timestamp=1732639244514Num uniques 859; 2024-11-26T16:40:44,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639244490, max timestamp=1732639244513Num uniques 859; 2024-11-26T16:40:44,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:40:44,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:40:44,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52758 deadline: 1732639304588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52792 deadline: 1732639304588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:44,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52818 deadline: 1732639304590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639304621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T16:40:44,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:44,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:44,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:44,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639304632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:44,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52758 deadline: 1732639304701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:44,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52792 deadline: 1732639304712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:44,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:44,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52818 deadline: 1732639304712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T16:40:44,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:44,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:44,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:44,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:44,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639304831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:44,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639304841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/7377c20524ce4bb78991200b10234ff1 2024-11-26T16:40:44,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:44,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52758 deadline: 1732639304922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:44,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52818 deadline: 1732639304932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:44,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52792 deadline: 1732639304932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:44,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:44,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:44,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/adf4a3d77bbf4ad0a2eb0ebdc296a0e9 is 50, key is test_row_0/C:col10/1732639243764/Put/seqid=0 2024-11-26T16:40:44,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T16:40:44,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:44,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:44,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:44,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:44,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741845_1021 (size=12001) 2024-11-26T16:40:45,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T16:40:45,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:45,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:45,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:45,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:45,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:45,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:45,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639305141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:45,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:45,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639305151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:45,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:45,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52758 deadline: 1732639305231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:45,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:45,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52818 deadline: 1732639305252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:45,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:45,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52792 deadline: 1732639305252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:45,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:45,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T16:40:45,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:45,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:45,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:45,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:45,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:45,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:45,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T16:40:45,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/adf4a3d77bbf4ad0a2eb0ebdc296a0e9 2024-11-26T16:40:45,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/611dc92837194c13ba0765006fa23697 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/611dc92837194c13ba0765006fa23697 2024-11-26T16:40:45,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/611dc92837194c13ba0765006fa23697, entries=200, sequenceid=47, filesize=14.0 K 2024-11-26T16:40:45,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/7377c20524ce4bb78991200b10234ff1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/7377c20524ce4bb78991200b10234ff1 2024-11-26T16:40:45,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T16:40:45,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:45,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:45,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:45,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:45,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:45,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/7377c20524ce4bb78991200b10234ff1, entries=150, sequenceid=47, filesize=11.7 K 2024-11-26T16:40:45,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/adf4a3d77bbf4ad0a2eb0ebdc296a0e9 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/adf4a3d77bbf4ad0a2eb0ebdc296a0e9 2024-11-26T16:40:45,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/adf4a3d77bbf4ad0a2eb0ebdc296a0e9, entries=150, sequenceid=47, filesize=11.7 K 2024-11-26T16:40:45,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 7c6dff940b7962766483b28d1008c073 in 1009ms, sequenceid=47, compaction requested=true 2024-11-26T16:40:45,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:45,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:40:45,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:45,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:40:45,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:45,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:40:45,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:40:45,450 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:45,453 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:45,454 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:45,456 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/B is initiating minor compaction (all files) 2024-11-26T16:40:45,457 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/B in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:45,457 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/281066a74a13447ab789ca5d9e973913, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/3ee7d070623e4f6b9245c08b4d23f881, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/7377c20524ce4bb78991200b10234ff1] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=35.2 K 2024-11-26T16:40:45,459 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 281066a74a13447ab789ca5d9e973913, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732639243197 2024-11-26T16:40:45,460 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ee7d070623e4f6b9245c08b4d23f881, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1732639243323 2024-11-26T16:40:45,461 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:45,461 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/A is initiating minor compaction (all files) 2024-11-26T16:40:45,461 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/A in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:45,461 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 7377c20524ce4bb78991200b10234ff1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732639243764 2024-11-26T16:40:45,461 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/57e15692b95d45f2b3c24dbdbeae9473, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/d76ff77803b34044acd81258000e957a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/611dc92837194c13ba0765006fa23697] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=42.0 K 2024-11-26T16:40:45,464 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57e15692b95d45f2b3c24dbdbeae9473, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732639243197 2024-11-26T16:40:45,465 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting d76ff77803b34044acd81258000e957a, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1732639243306 2024-11-26T16:40:45,465 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 611dc92837194c13ba0765006fa23697, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732639243764 2024-11-26T16:40:45,537 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#A#compaction#9 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:45,537 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#B#compaction#10 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:45,538 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/a446dfb2db594cde8c13c0d2d12c027d is 50, key is test_row_0/A:col10/1732639243764/Put/seqid=0 2024-11-26T16:40:45,538 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/8e4f5faebd834c2eb2bc3cea0ee41bc4 is 50, key is test_row_0/B:col10/1732639243764/Put/seqid=0 2024-11-26T16:40:45,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741847_1023 (size=12104) 2024-11-26T16:40:45,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T16:40:45,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:45,579 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-11-26T16:40:45,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:40:45,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:40:45,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:40:45,580 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/8e4f5faebd834c2eb2bc3cea0ee41bc4 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/8e4f5faebd834c2eb2bc3cea0ee41bc4 2024-11-26T16:40:45,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:40:45,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:40:45,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:40:45,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741846_1022 (size=12104) 2024-11-26T16:40:45,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/2877dec8bb8c48d686365672c33daff4 is 50, key is test_row_0/A:col10/1732639244575/Put/seqid=0 2024-11-26T16:40:45,640 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/B of 7c6dff940b7962766483b28d1008c073 into 8e4f5faebd834c2eb2bc3cea0ee41bc4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:45,640 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:45,643 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/B, priority=13, startTime=1732639245447; duration=0sec 2024-11-26T16:40:45,644 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:40:45,644 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:B 2024-11-26T16:40:45,645 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:45,650 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:45,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741848_1024 (size=12001) 2024-11-26T16:40:45,650 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/C is initiating minor compaction (all files) 2024-11-26T16:40:45,650 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/C in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:45,651 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/49b91840b6ef40ea802b12272e8a2c4c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/4f3d52925f2a40f78c7261c403708cb3, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/adf4a3d77bbf4ad0a2eb0ebdc296a0e9] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=35.2 K 2024-11-26T16:40:45,651 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/2877dec8bb8c48d686365672c33daff4 2024-11-26T16:40:45,652 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 49b91840b6ef40ea802b12272e8a2c4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732639243197 2024-11-26T16:40:45,653 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f3d52925f2a40f78c7261c403708cb3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1732639243323 2024-11-26T16:40:45,654 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting adf4a3d77bbf4ad0a2eb0ebdc296a0e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732639243764 2024-11-26T16:40:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:45,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:45,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/cf0acafc4567495aadd676b98fc8a63d is 50, key is test_row_0/B:col10/1732639244575/Put/seqid=0 2024-11-26T16:40:45,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741849_1025 (size=12001) 2024-11-26T16:40:45,703 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/cf0acafc4567495aadd676b98fc8a63d 2024-11-26T16:40:45,705 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#C#compaction#13 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:45,706 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/0ec213e08b904952839187f8da3a083d is 50, key is test_row_0/C:col10/1732639243764/Put/seqid=0 2024-11-26T16:40:45,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639305728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:45,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639305729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:45,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52758 deadline: 1732639305741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:45,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:45,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741850_1026 (size=12104) 2024-11-26T16:40:45,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/386d7a5c1765440688692c716df36723 is 50, key is test_row_0/C:col10/1732639244575/Put/seqid=0 2024-11-26T16:40:45,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52818 deadline: 1732639305762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:45,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52792 deadline: 1732639305762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:45,779 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/0ec213e08b904952839187f8da3a083d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/0ec213e08b904952839187f8da3a083d 2024-11-26T16:40:45,794 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/C of 7c6dff940b7962766483b28d1008c073 into 0ec213e08b904952839187f8da3a083d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:45,794 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:45,794 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/C, priority=13, startTime=1732639245447; duration=0sec 2024-11-26T16:40:45,794 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:45,795 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:C 2024-11-26T16:40:45,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741851_1027 (size=12001) 2024-11-26T16:40:45,805 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/386d7a5c1765440688692c716df36723 2024-11-26T16:40:45,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/2877dec8bb8c48d686365672c33daff4 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2877dec8bb8c48d686365672c33daff4 2024-11-26T16:40:45,842 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2877dec8bb8c48d686365672c33daff4, entries=150, sequenceid=82, filesize=11.7 K 2024-11-26T16:40:45,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/cf0acafc4567495aadd676b98fc8a63d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/cf0acafc4567495aadd676b98fc8a63d 2024-11-26T16:40:45,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639305852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:45,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639305852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:45,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:45,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:45,871 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/cf0acafc4567495aadd676b98fc8a63d, entries=150, sequenceid=82, filesize=11.7 K 2024-11-26T16:40:45,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/386d7a5c1765440688692c716df36723 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/386d7a5c1765440688692c716df36723 2024-11-26T16:40:45,898 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/386d7a5c1765440688692c716df36723, entries=150, sequenceid=82, filesize=11.7 K 2024-11-26T16:40:45,900 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=60.38 KB/61830 for 7c6dff940b7962766483b28d1008c073 in 320ms, sequenceid=82, compaction requested=false 2024-11-26T16:40:45,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:45,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:45,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-26T16:40:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-26T16:40:45,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T16:40:45,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7480 sec 2024-11-26T16:40:45,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.7870 sec 2024-11-26T16:40:46,029 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/a446dfb2db594cde8c13c0d2d12c027d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/a446dfb2db594cde8c13c0d2d12c027d 2024-11-26T16:40:46,054 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/A of 7c6dff940b7962766483b28d1008c073 into a446dfb2db594cde8c13c0d2d12c027d(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:46,054 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:46,055 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/A, priority=13, startTime=1732639245441; duration=0sec 2024-11-26T16:40:46,055 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:46,055 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:A 2024-11-26T16:40:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:46,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T16:40:46,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:40:46,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:46,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:40:46,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:46,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:40:46,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:46,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/2c24f84ea6ff4e688528b8c8c735ed22 is 50, key is test_row_0/A:col10/1732639245721/Put/seqid=0 2024-11-26T16:40:46,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741852_1028 (size=14341) 2024-11-26T16:40:46,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/2c24f84ea6ff4e688528b8c8c735ed22 2024-11-26T16:40:46,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/e2b33f19efdb4aac81fcc6bbf81e9b2b is 50, key is test_row_0/B:col10/1732639245721/Put/seqid=0 2024-11-26T16:40:46,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639306177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:46,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:46,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741853_1029 (size=12001) 2024-11-26T16:40:46,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/e2b33f19efdb4aac81fcc6bbf81e9b2b 2024-11-26T16:40:46,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639306182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:46,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:46,203 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T16:40:46,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/a66277bb1724469b949fe1149616c7b0 is 50, key is test_row_0/C:col10/1732639245721/Put/seqid=0 2024-11-26T16:40:46,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741854_1030 (size=12001) 2024-11-26T16:40:46,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/a66277bb1724469b949fe1149616c7b0 2024-11-26T16:40:46,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/2c24f84ea6ff4e688528b8c8c735ed22 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2c24f84ea6ff4e688528b8c8c735ed22 2024-11-26T16:40:46,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2c24f84ea6ff4e688528b8c8c735ed22, entries=200, sequenceid=98, filesize=14.0 K 2024-11-26T16:40:46,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/e2b33f19efdb4aac81fcc6bbf81e9b2b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/e2b33f19efdb4aac81fcc6bbf81e9b2b 2024-11-26T16:40:46,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/e2b33f19efdb4aac81fcc6bbf81e9b2b, entries=150, sequenceid=98, filesize=11.7 K 2024-11-26T16:40:46,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/a66277bb1724469b949fe1149616c7b0 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/a66277bb1724469b949fe1149616c7b0 2024-11-26T16:40:46,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/a66277bb1724469b949fe1149616c7b0, entries=150, sequenceid=98, filesize=11.7 K 2024-11-26T16:40:46,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639306292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:46,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:46,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 7c6dff940b7962766483b28d1008c073 in 226ms, sequenceid=98, compaction requested=true 2024-11-26T16:40:46,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:46,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:40:46,302 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:46,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:46,303 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:46,306 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:46,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:40:46,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:46,306 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/B is initiating minor compaction (all files) 2024-11-26T16:40:46,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:40:46,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:40:46,306 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/B in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:46,306 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/8e4f5faebd834c2eb2bc3cea0ee41bc4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/cf0acafc4567495aadd676b98fc8a63d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/e2b33f19efdb4aac81fcc6bbf81e9b2b] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=35.3 K 2024-11-26T16:40:46,307 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:46,307 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/A is initiating minor compaction (all files) 2024-11-26T16:40:46,307 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/A in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:46,307 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/a446dfb2db594cde8c13c0d2d12c027d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2877dec8bb8c48d686365672c33daff4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2c24f84ea6ff4e688528b8c8c735ed22] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=37.5 K 2024-11-26T16:40:46,310 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e4f5faebd834c2eb2bc3cea0ee41bc4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732639243764 2024-11-26T16:40:46,310 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting a446dfb2db594cde8c13c0d2d12c027d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732639243764 2024-11-26T16:40:46,311 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting cf0acafc4567495aadd676b98fc8a63d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732639244575 2024-11-26T16:40:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:46,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T16:40:46,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:40:46,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:46,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:40:46,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:46,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:40:46,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:46,315 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2877dec8bb8c48d686365672c33daff4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732639244575 2024-11-26T16:40:46,318 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c24f84ea6ff4e688528b8c8c735ed22, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732639245706 2024-11-26T16:40:46,320 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting e2b33f19efdb4aac81fcc6bbf81e9b2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732639245706 2024-11-26T16:40:46,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/0407a74ddcf94ba99b7c8c5ca19b02fb is 50, key is test_row_0/A:col10/1732639246173/Put/seqid=0 2024-11-26T16:40:46,355 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:46,356 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/d3a965695ec64677ac94c6fe8adaf89b is 50, key is test_row_0/A:col10/1732639245721/Put/seqid=0 2024-11-26T16:40:46,363 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#B#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:46,364 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/001fda3ce6704a6fb82061ef2f51e433 is 50, key is test_row_0/B:col10/1732639245721/Put/seqid=0 2024-11-26T16:40:46,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741855_1031 (size=12001) 2024-11-26T16:40:46,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/0407a74ddcf94ba99b7c8c5ca19b02fb 2024-11-26T16:40:46,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639306416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:46,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741856_1032 (size=12207) 2024-11-26T16:40:46,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741857_1033 (size=12207) 2024-11-26T16:40:46,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/5e2a95db35244527827fd0f2df4d1602 is 50, key is test_row_0/B:col10/1732639246173/Put/seqid=0 2024-11-26T16:40:46,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741858_1034 (size=12001) 2024-11-26T16:40:46,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/5e2a95db35244527827fd0f2df4d1602 2024-11-26T16:40:46,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/955df0cba5074116adcae4a7ac22d5f2 is 50, key is test_row_0/C:col10/1732639246173/Put/seqid=0 2024-11-26T16:40:46,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639306512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:46,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741859_1035 (size=12001) 2024-11-26T16:40:46,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:46,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639306531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:46,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639306742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:46,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52758 deadline: 1732639306752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:46,760 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_0' in 7c6dff940b7962766483b28d1008c073 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:40:46,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:46,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:46,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52818 deadline: 1732639306781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:46,784 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in 7c6dff940b7962766483b28d1008c073 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:40:46,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:46,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52792 deadline: 1732639306791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:46,796 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_0' in 7c6dff940b7962766483b28d1008c073 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:40:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:46,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:46,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639306821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:46,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:46,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:46,861 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/d3a965695ec64677ac94c6fe8adaf89b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/d3a965695ec64677ac94c6fe8adaf89b 2024-11-26T16:40:46,862 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/001fda3ce6704a6fb82061ef2f51e433 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/001fda3ce6704a6fb82061ef2f51e433 2024-11-26T16:40:46,875 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/B of 7c6dff940b7962766483b28d1008c073 into 001fda3ce6704a6fb82061ef2f51e433(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:46,877 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:46,878 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/B, priority=13, startTime=1732639246303; duration=0sec 2024-11-26T16:40:46,878 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:40:46,878 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:B 2024-11-26T16:40:46,878 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:46,882 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:46,882 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/C is initiating minor compaction (all files) 2024-11-26T16:40:46,882 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/C in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:46,883 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/0ec213e08b904952839187f8da3a083d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/386d7a5c1765440688692c716df36723, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/a66277bb1724469b949fe1149616c7b0] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=35.3 K 2024-11-26T16:40:46,884 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/A of 7c6dff940b7962766483b28d1008c073 into d3a965695ec64677ac94c6fe8adaf89b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:46,884 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ec213e08b904952839187f8da3a083d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732639243764 2024-11-26T16:40:46,884 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:46,884 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/A, priority=13, startTime=1732639246302; duration=0sec 2024-11-26T16:40:46,885 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:46,885 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:A 2024-11-26T16:40:46,889 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 386d7a5c1765440688692c716df36723, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732639244575 2024-11-26T16:40:46,890 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a66277bb1724469b949fe1149616c7b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732639245706 2024-11-26T16:40:46,915 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#C#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:46,916 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/e9e5b95057e14d898542688396da6408 is 50, key is test_row_0/C:col10/1732639245721/Put/seqid=0 2024-11-26T16:40:46,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/955df0cba5074116adcae4a7ac22d5f2 2024-11-26T16:40:46,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741860_1036 (size=12207) 2024-11-26T16:40:46,954 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T16:40:46,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/0407a74ddcf94ba99b7c8c5ca19b02fb as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/0407a74ddcf94ba99b7c8c5ca19b02fb 2024-11-26T16:40:46,967 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/e9e5b95057e14d898542688396da6408 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/e9e5b95057e14d898542688396da6408 2024-11-26T16:40:46,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/0407a74ddcf94ba99b7c8c5ca19b02fb, entries=150, sequenceid=118, filesize=11.7 K 2024-11-26T16:40:46,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/5e2a95db35244527827fd0f2df4d1602 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/5e2a95db35244527827fd0f2df4d1602 2024-11-26T16:40:47,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/5e2a95db35244527827fd0f2df4d1602, entries=150, sequenceid=118, filesize=11.7 K 2024-11-26T16:40:47,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/955df0cba5074116adcae4a7ac22d5f2 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/955df0cba5074116adcae4a7ac22d5f2 2024-11-26T16:40:47,010 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/C of 7c6dff940b7962766483b28d1008c073 into e9e5b95057e14d898542688396da6408(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:47,010 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:47,010 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/C, priority=13, startTime=1732639246306; duration=0sec 2024-11-26T16:40:47,010 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:47,010 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:C 2024-11-26T16:40:47,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/955df0cba5074116adcae4a7ac22d5f2, entries=150, sequenceid=118, filesize=11.7 K 2024-11-26T16:40:47,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 7c6dff940b7962766483b28d1008c073 in 712ms, sequenceid=118, compaction requested=false 2024-11-26T16:40:47,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:47,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T16:40:47,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:40:47,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:47,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:40:47,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:47,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:40:47,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:47,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/dd6b3892469a45aa8cccc1143bb2f773 is 50, key is test_row_0/A:col10/1732639246404/Put/seqid=0 2024-11-26T16:40:47,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741861_1037 (size=14441) 2024-11-26T16:40:47,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:47,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639307221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:47,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:47,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:47,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T16:40:47,312 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:40:47,313 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:40:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-26T16:40:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T16:40:47,328 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:40:47,331 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:40:47,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:40:47,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639307343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:47,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:47,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:47,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639307343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:47,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:47,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:47,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T16:40:47,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T16:40:47,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:47,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:47,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:47,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:47,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:47,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/dd6b3892469a45aa8cccc1143bb2f773 2024-11-26T16:40:47,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/7f0445b2c3a94661a02dbdc0b8e5c971 is 50, key is test_row_0/B:col10/1732639246404/Put/seqid=0 2024-11-26T16:40:47,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639307552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:47,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741862_1038 (size=12051) 2024-11-26T16:40:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T16:40:47,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T16:40:47,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:47,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:47,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:47,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:47,782 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T16:40:47,783 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-26T16:40:47,785 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T16:40:47,785 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-26T16:40:47,786 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-26T16:40:47,786 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-26T16:40:47,787 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-26T16:40:47,787 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-26T16:40:47,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T16:40:47,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:47,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:47,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:47,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:47,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:47,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639307871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:47,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:47,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:47,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T16:40:47,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T16:40:47,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:47,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:47,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:47,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:47,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:47,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/7f0445b2c3a94661a02dbdc0b8e5c971 2024-11-26T16:40:47,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/af859a2f1e634b2fa55e71a69ed15e4a is 50, key is test_row_0/C:col10/1732639246404/Put/seqid=0 2024-11-26T16:40:48,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741863_1039 (size=12051) 2024-11-26T16:40:48,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/af859a2f1e634b2fa55e71a69ed15e4a 2024-11-26T16:40:48,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/dd6b3892469a45aa8cccc1143bb2f773 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/dd6b3892469a45aa8cccc1143bb2f773 2024-11-26T16:40:48,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/dd6b3892469a45aa8cccc1143bb2f773, entries=200, sequenceid=134, filesize=14.1 K 2024-11-26T16:40:48,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/7f0445b2c3a94661a02dbdc0b8e5c971 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/7f0445b2c3a94661a02dbdc0b8e5c971 2024-11-26T16:40:48,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/7f0445b2c3a94661a02dbdc0b8e5c971, entries=150, sequenceid=134, filesize=11.8 K 2024-11-26T16:40:48,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/af859a2f1e634b2fa55e71a69ed15e4a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/af859a2f1e634b2fa55e71a69ed15e4a 2024-11-26T16:40:48,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/af859a2f1e634b2fa55e71a69ed15e4a, entries=150, sequenceid=134, filesize=11.8 K 2024-11-26T16:40:48,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 7c6dff940b7962766483b28d1008c073 in 998ms, sequenceid=134, compaction requested=true 2024-11-26T16:40:48,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:48,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:40:48,063 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:48,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:48,063 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:48,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:40:48,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:48,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:40:48,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:40:48,066 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:48,066 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:48,066 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/A is initiating minor compaction (all files) 2024-11-26T16:40:48,066 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/B is initiating minor compaction (all files) 2024-11-26T16:40:48,066 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/B in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:48,066 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/A in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:48,066 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/001fda3ce6704a6fb82061ef2f51e433, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/5e2a95db35244527827fd0f2df4d1602, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/7f0445b2c3a94661a02dbdc0b8e5c971] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=35.4 K 2024-11-26T16:40:48,066 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/d3a965695ec64677ac94c6fe8adaf89b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/0407a74ddcf94ba99b7c8c5ca19b02fb, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/dd6b3892469a45aa8cccc1143bb2f773] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=37.7 K 2024-11-26T16:40:48,067 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3a965695ec64677ac94c6fe8adaf89b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732639245706 2024-11-26T16:40:48,068 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 001fda3ce6704a6fb82061ef2f51e433, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732639245706 2024-11-26T16:40:48,068 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0407a74ddcf94ba99b7c8c5ca19b02fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732639246170 2024-11-26T16:40:48,068 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e2a95db35244527827fd0f2df4d1602, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732639246170 2024-11-26T16:40:48,069 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd6b3892469a45aa8cccc1143bb2f773, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732639246396 2024-11-26T16:40:48,069 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f0445b2c3a94661a02dbdc0b8e5c971, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732639246396 2024-11-26T16:40:48,092 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#B#compaction#27 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:48,093 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/cfeb94040aa74e43a61fa983e903fece is 50, key is test_row_0/B:col10/1732639246404/Put/seqid=0 2024-11-26T16:40:48,100 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#A#compaction#28 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:48,101 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/f81739ddcb894addb8be6703d80499ce is 50, key is test_row_0/A:col10/1732639246404/Put/seqid=0 2024-11-26T16:40:48,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T16:40:48,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:48,118 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T16:40:48,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:40:48,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:48,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:40:48,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:48,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:40:48,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:48,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741864_1040 (size=12359) 2024-11-26T16:40:48,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741865_1041 (size=12359) 2024-11-26T16:40:48,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/98f8196b7d4d4651a5848c1151d324de is 50, key is test_row_0/A:col10/1732639247131/Put/seqid=0 2024-11-26T16:40:48,142 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/f81739ddcb894addb8be6703d80499ce as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/f81739ddcb894addb8be6703d80499ce 2024-11-26T16:40:48,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741866_1042 (size=12151) 2024-11-26T16:40:48,157 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/A of 7c6dff940b7962766483b28d1008c073 into f81739ddcb894addb8be6703d80499ce(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:48,157 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:48,157 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/A, priority=13, startTime=1732639248063; duration=0sec 2024-11-26T16:40:48,157 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:40:48,157 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:A 2024-11-26T16:40:48,157 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:48,159 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:48,159 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/C is initiating minor compaction (all files) 2024-11-26T16:40:48,160 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/C in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:48,160 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/e9e5b95057e14d898542688396da6408, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/955df0cba5074116adcae4a7ac22d5f2, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/af859a2f1e634b2fa55e71a69ed15e4a] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=35.4 K 2024-11-26T16:40:48,163 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9e5b95057e14d898542688396da6408, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732639245706 2024-11-26T16:40:48,163 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 955df0cba5074116adcae4a7ac22d5f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732639246170 2024-11-26T16:40:48,164 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting af859a2f1e634b2fa55e71a69ed15e4a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732639246396 2024-11-26T16:40:48,179 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#C#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:48,180 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/ac6a6d22961440ea9027dc6e5d739f81 is 50, key is test_row_0/C:col10/1732639246404/Put/seqid=0 2024-11-26T16:40:48,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741867_1043 (size=12359) 2024-11-26T16:40:48,210 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/ac6a6d22961440ea9027dc6e5d739f81 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/ac6a6d22961440ea9027dc6e5d739f81 2024-11-26T16:40:48,224 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/C of 7c6dff940b7962766483b28d1008c073 into ac6a6d22961440ea9027dc6e5d739f81(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:48,224 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:48,224 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/C, priority=13, startTime=1732639248063; duration=0sec 2024-11-26T16:40:48,224 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:48,224 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:C 2024-11-26T16:40:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:48,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:48,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639308395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:48,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:48,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639308396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:48,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:48,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:48,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T16:40:48,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639308511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:48,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:48,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639308511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:48,532 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/cfeb94040aa74e43a61fa983e903fece as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/cfeb94040aa74e43a61fa983e903fece 2024-11-26T16:40:48,543 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/B of 7c6dff940b7962766483b28d1008c073 into cfeb94040aa74e43a61fa983e903fece(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:48,543 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:48,543 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/B, priority=13, startTime=1732639248063; duration=0sec 2024-11-26T16:40:48,543 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:48,543 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:B 2024-11-26T16:40:48,553 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/98f8196b7d4d4651a5848c1151d324de 2024-11-26T16:40:48,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/92bb23045ab64cc99acd3fccc3d58c4b is 50, key is test_row_0/B:col10/1732639247131/Put/seqid=0 2024-11-26T16:40:48,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741868_1044 (size=12151) 2024-11-26T16:40:48,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:48,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639308731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:48,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:48,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639308732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:48,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:48,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:48,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:48,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52758 deadline: 1732639308783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:48,788 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_0' in 7c6dff940b7962766483b28d1008c073 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:40:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:48,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:48,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52818 deadline: 1732639308801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:48,805 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in 7c6dff940b7962766483b28d1008c073 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:40:48,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:48,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:48,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:48,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52792 deadline: 1732639308811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:48,816 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_0' in 7c6dff940b7962766483b28d1008c073 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:40:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:48,995 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/92bb23045ab64cc99acd3fccc3d58c4b 2024-11-26T16:40:49,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/377ea084cce24b609a770f9816083cc8 is 50, key is test_row_0/C:col10/1732639247131/Put/seqid=0 2024-11-26T16:40:49,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741869_1045 (size=12151) 2024-11-26T16:40:49,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:49,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639309041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:49,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:49,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:49,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639309042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:49,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:49,421 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/377ea084cce24b609a770f9816083cc8 2024-11-26T16:40:49,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/98f8196b7d4d4651a5848c1151d324de as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/98f8196b7d4d4651a5848c1151d324de 2024-11-26T16:40:49,440 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/98f8196b7d4d4651a5848c1151d324de, entries=150, sequenceid=153, filesize=11.9 K 2024-11-26T16:40:49,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/92bb23045ab64cc99acd3fccc3d58c4b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/92bb23045ab64cc99acd3fccc3d58c4b 2024-11-26T16:40:49,450 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/92bb23045ab64cc99acd3fccc3d58c4b, entries=150, sequenceid=153, filesize=11.9 K 2024-11-26T16:40:49,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/377ea084cce24b609a770f9816083cc8 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/377ea084cce24b609a770f9816083cc8 2024-11-26T16:40:49,464 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/377ea084cce24b609a770f9816083cc8, entries=150, sequenceid=153, filesize=11.9 K 2024-11-26T16:40:49,465 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 7c6dff940b7962766483b28d1008c073 in 1348ms, sequenceid=153, compaction requested=false 2024-11-26T16:40:49,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:49,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:49,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-26T16:40:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-26T16:40:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T16:40:49,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-26T16:40:49,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1370 sec 2024-11-26T16:40:49,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.1600 sec 2024-11-26T16:40:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:49,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T16:40:49,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:40:49,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:49,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:40:49,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:49,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:40:49,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:49,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/c4dc25060d1b436eaefef86cbd040be1 is 50, key is test_row_0/A:col10/1732639248392/Put/seqid=0 2024-11-26T16:40:49,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741870_1046 (size=14541) 2024-11-26T16:40:49,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/c4dc25060d1b436eaefef86cbd040be1 2024-11-26T16:40:49,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/41579a9c45ae4dc2a0992281755fdb0d is 50, key is test_row_0/B:col10/1732639248392/Put/seqid=0 2024-11-26T16:40:49,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639309606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:49,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:49,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639309613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:49,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741871_1047 (size=12151) 2024-11-26T16:40:49,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/41579a9c45ae4dc2a0992281755fdb0d 2024-11-26T16:40:49,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/d8e4b6769a694820b1fffd961906a086 is 50, key is test_row_0/C:col10/1732639248392/Put/seqid=0 2024-11-26T16:40:49,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741872_1048 (size=12151) 2024-11-26T16:40:49,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639309721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:49,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639309722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:49,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:49,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:49,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:49,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639309932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:49,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639309932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:50,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/d8e4b6769a694820b1fffd961906a086 2024-11-26T16:40:50,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/c4dc25060d1b436eaefef86cbd040be1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/c4dc25060d1b436eaefef86cbd040be1 2024-11-26T16:40:50,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/c4dc25060d1b436eaefef86cbd040be1, entries=200, sequenceid=172, filesize=14.2 K 2024-11-26T16:40:50,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/41579a9c45ae4dc2a0992281755fdb0d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/41579a9c45ae4dc2a0992281755fdb0d 2024-11-26T16:40:50,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/41579a9c45ae4dc2a0992281755fdb0d, entries=150, sequenceid=172, filesize=11.9 K 2024-11-26T16:40:50,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/d8e4b6769a694820b1fffd961906a086 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/d8e4b6769a694820b1fffd961906a086 2024-11-26T16:40:50,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/d8e4b6769a694820b1fffd961906a086, entries=150, sequenceid=172, filesize=11.9 K 2024-11-26T16:40:50,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 7c6dff940b7962766483b28d1008c073 in 528ms, sequenceid=172, compaction requested=true 2024-11-26T16:40:50,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:50,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:40:50,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:50,081 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:50,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:40:50,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:50,081 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:50,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:40:50,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:40:50,084 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:50,084 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/B is initiating minor compaction (all files) 2024-11-26T16:40:50,084 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39051 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:50,084 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/B in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:50,084 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/A is initiating minor compaction (all files) 2024-11-26T16:40:50,085 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/cfeb94040aa74e43a61fa983e903fece, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/92bb23045ab64cc99acd3fccc3d58c4b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/41579a9c45ae4dc2a0992281755fdb0d] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=35.8 K 2024-11-26T16:40:50,085 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/A in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:50,085 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/f81739ddcb894addb8be6703d80499ce, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/98f8196b7d4d4651a5848c1151d324de, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/c4dc25060d1b436eaefef86cbd040be1] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=38.1 K 2024-11-26T16:40:50,085 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting cfeb94040aa74e43a61fa983e903fece, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732639246396 2024-11-26T16:40:50,086 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting f81739ddcb894addb8be6703d80499ce, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732639246396 2024-11-26T16:40:50,086 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 92bb23045ab64cc99acd3fccc3d58c4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732639247131 2024-11-26T16:40:50,086 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98f8196b7d4d4651a5848c1151d324de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732639247131 2024-11-26T16:40:50,086 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 41579a9c45ae4dc2a0992281755fdb0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732639248392 2024-11-26T16:40:50,088 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4dc25060d1b436eaefef86cbd040be1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732639248392 2024-11-26T16:40:50,100 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:50,101 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/1523339a76794901a640d8d881279578 is 50, key is test_row_0/B:col10/1732639248392/Put/seqid=0 2024-11-26T16:40:50,111 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#A#compaction#37 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:50,111 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/4fe70b3fc2754ae89c04e620aa4cbd7a is 50, key is test_row_0/A:col10/1732639248392/Put/seqid=0 2024-11-26T16:40:50,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741873_1049 (size=12561) 2024-11-26T16:40:50,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741874_1050 (size=12561) 2024-11-26T16:40:50,158 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/4fe70b3fc2754ae89c04e620aa4cbd7a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/4fe70b3fc2754ae89c04e620aa4cbd7a 2024-11-26T16:40:50,170 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/A of 7c6dff940b7962766483b28d1008c073 into 4fe70b3fc2754ae89c04e620aa4cbd7a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:50,170 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:50,170 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/A, priority=13, startTime=1732639250081; duration=0sec 2024-11-26T16:40:50,170 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:40:50,171 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:A 2024-11-26T16:40:50,171 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:50,172 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:50,172 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/C is initiating minor compaction (all files) 2024-11-26T16:40:50,172 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/C in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:50,173 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/ac6a6d22961440ea9027dc6e5d739f81, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/377ea084cce24b609a770f9816083cc8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/d8e4b6769a694820b1fffd961906a086] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=35.8 K 2024-11-26T16:40:50,173 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac6a6d22961440ea9027dc6e5d739f81, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732639246396 2024-11-26T16:40:50,174 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 377ea084cce24b609a770f9816083cc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732639247131 2024-11-26T16:40:50,174 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8e4b6769a694820b1fffd961906a086, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732639248392 2024-11-26T16:40:50,187 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#C#compaction#38 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:50,187 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/2340fc53e7a84c70930207e945b8357c is 50, key is test_row_0/C:col10/1732639248392/Put/seqid=0 2024-11-26T16:40:50,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741875_1051 (size=12561) 2024-11-26T16:40:50,204 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/2340fc53e7a84c70930207e945b8357c as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/2340fc53e7a84c70930207e945b8357c 2024-11-26T16:40:50,217 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/C of 7c6dff940b7962766483b28d1008c073 into 2340fc53e7a84c70930207e945b8357c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:50,217 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:50,217 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/C, priority=13, startTime=1732639250081; duration=0sec 2024-11-26T16:40:50,217 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:50,217 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:C 2024-11-26T16:40:50,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:50,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T16:40:50,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:40:50,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:50,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:40:50,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:50,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:40:50,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:50,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/e40c9a7512694a53b93cdcda0112209e is 50, key is test_row_0/A:col10/1732639249607/Put/seqid=0 2024-11-26T16:40:50,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741876_1052 (size=14541) 2024-11-26T16:40:50,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:50,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639310284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:50,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:50,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:50,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:50,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:50,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639310286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:50,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:50,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:50,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:50,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:50,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639310392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:50,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:50,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639310392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:50,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:50,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:50,548 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/1523339a76794901a640d8d881279578 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/1523339a76794901a640d8d881279578 2024-11-26T16:40:50,558 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/B of 7c6dff940b7962766483b28d1008c073 into 1523339a76794901a640d8d881279578(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:50,558 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:50,558 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/B, priority=13, startTime=1732639250081; duration=0sec 2024-11-26T16:40:50,558 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:50,558 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:B 2024-11-26T16:40:50,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:50,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639310602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:50,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:50,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:50,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:50,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639310611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:50,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:50,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:50,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:50,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/e40c9a7512694a53b93cdcda0112209e 2024-11-26T16:40:50,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/e19262cbfcdd44bfbbcbfb4828e98542 is 50, key is test_row_0/B:col10/1732639249607/Put/seqid=0 2024-11-26T16:40:50,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741877_1053 (size=12151) 2024-11-26T16:40:50,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/e19262cbfcdd44bfbbcbfb4828e98542 2024-11-26T16:40:50,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/b44e0172af184bf8800b70fc3f3c9462 is 50, key is test_row_0/C:col10/1732639249607/Put/seqid=0 2024-11-26T16:40:50,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741878_1054 (size=12151) 2024-11-26T16:40:50,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/b44e0172af184bf8800b70fc3f3c9462 2024-11-26T16:40:50,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/e40c9a7512694a53b93cdcda0112209e as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/e40c9a7512694a53b93cdcda0112209e 2024-11-26T16:40:50,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/e40c9a7512694a53b93cdcda0112209e, entries=200, sequenceid=192, filesize=14.2 K 2024-11-26T16:40:50,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/e19262cbfcdd44bfbbcbfb4828e98542 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/e19262cbfcdd44bfbbcbfb4828e98542 2024-11-26T16:40:50,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/e19262cbfcdd44bfbbcbfb4828e98542, entries=150, sequenceid=192, filesize=11.9 K 2024-11-26T16:40:50,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/b44e0172af184bf8800b70fc3f3c9462 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/b44e0172af184bf8800b70fc3f3c9462 2024-11-26T16:40:50,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/b44e0172af184bf8800b70fc3f3c9462, entries=150, sequenceid=192, filesize=11.9 K 2024-11-26T16:40:50,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 7c6dff940b7962766483b28d1008c073 in 524ms, sequenceid=192, compaction requested=false 2024-11-26T16:40:50,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:50,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T16:40:50,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:40:50,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:50,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:40:50,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:50,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:40:50,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:50,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/43fd5fe3ce7f42b48f9518148937adb6 is 50, key is test_row_0/A:col10/1732639250916/Put/seqid=0 2024-11-26T16:40:50,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741879_1055 (size=12151) 2024-11-26T16:40:51,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639311025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:51,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639311026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:51,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639311142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:51,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639311142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:51,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/43fd5fe3ce7f42b48f9518148937adb6 2024-11-26T16:40:51,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639311351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:51,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:51,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:51,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:51,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639311352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:51,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:51,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:51,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:51,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/18c81b7c41d346fe9f8b9dfb107b2d4d is 50, key is test_row_0/B:col10/1732639250916/Put/seqid=0 2024-11-26T16:40:51,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741880_1056 (size=12151) 2024-11-26T16:40:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T16:40:51,491 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:40:51,492 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:40:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-26T16:40:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T16:40:51,494 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:40:51,496 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:40:51,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:40:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T16:40:51,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T16:40:51,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:51,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:51,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:51,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:51,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:51,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639311661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:51,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639311661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:51,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/18c81b7c41d346fe9f8b9dfb107b2d4d 2024-11-26T16:40:51,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/f834988a7d7b441cb01b807685a610fb is 50, key is test_row_0/C:col10/1732639250916/Put/seqid=0 2024-11-26T16:40:51,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741881_1057 (size=12151) 2024-11-26T16:40:51,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/f834988a7d7b441cb01b807685a610fb 2024-11-26T16:40:51,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T16:40:51,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/43fd5fe3ce7f42b48f9518148937adb6 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/43fd5fe3ce7f42b48f9518148937adb6 2024-11-26T16:40:51,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:51,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:51,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:51,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:51,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T16:40:51,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/43fd5fe3ce7f42b48f9518148937adb6, entries=150, sequenceid=208, filesize=11.9 K 2024-11-26T16:40:51,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/18c81b7c41d346fe9f8b9dfb107b2d4d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/18c81b7c41d346fe9f8b9dfb107b2d4d 2024-11-26T16:40:51,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/18c81b7c41d346fe9f8b9dfb107b2d4d, entries=150, sequenceid=208, filesize=11.9 K 2024-11-26T16:40:51,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/f834988a7d7b441cb01b807685a610fb as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/f834988a7d7b441cb01b807685a610fb 2024-11-26T16:40:51,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/f834988a7d7b441cb01b807685a610fb, entries=150, sequenceid=208, filesize=11.9 K 2024-11-26T16:40:51,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 7c6dff940b7962766483b28d1008c073 in 912ms, sequenceid=208, compaction requested=true 2024-11-26T16:40:51,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:51,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:40:51,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:51,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:40:51,833 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:51,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:51,833 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:51,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:40:51,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:40:51,834 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:51,834 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:51,835 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/A is initiating minor compaction (all files) 2024-11-26T16:40:51,835 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/B is initiating minor compaction (all files) 2024-11-26T16:40:51,835 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/A in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:51,835 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/B in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:51,835 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/4fe70b3fc2754ae89c04e620aa4cbd7a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/e40c9a7512694a53b93cdcda0112209e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/43fd5fe3ce7f42b48f9518148937adb6] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=38.3 K 2024-11-26T16:40:51,835 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/1523339a76794901a640d8d881279578, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/e19262cbfcdd44bfbbcbfb4828e98542, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/18c81b7c41d346fe9f8b9dfb107b2d4d] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=36.0 K 2024-11-26T16:40:51,835 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fe70b3fc2754ae89c04e620aa4cbd7a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732639248392 2024-11-26T16:40:51,835 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 1523339a76794901a640d8d881279578, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732639248392 2024-11-26T16:40:51,836 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting e40c9a7512694a53b93cdcda0112209e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732639249601 2024-11-26T16:40:51,837 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting e19262cbfcdd44bfbbcbfb4828e98542, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732639249607 2024-11-26T16:40:51,837 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43fd5fe3ce7f42b48f9518148937adb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732639250257 2024-11-26T16:40:51,838 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 18c81b7c41d346fe9f8b9dfb107b2d4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732639250257 2024-11-26T16:40:51,849 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#B#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:51,850 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/9522f6f6bb2f4c45abb016e514972c03 is 50, key is test_row_0/B:col10/1732639250916/Put/seqid=0 2024-11-26T16:40:51,852 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#A#compaction#46 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:51,853 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/05a16c2dc03c47138adb361e67f37241 is 50, key is test_row_0/A:col10/1732639250916/Put/seqid=0 2024-11-26T16:40:51,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741882_1058 (size=12663) 2024-11-26T16:40:51,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741883_1059 (size=12663) 2024-11-26T16:40:51,883 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/05a16c2dc03c47138adb361e67f37241 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/05a16c2dc03c47138adb361e67f37241 2024-11-26T16:40:51,891 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/A of 7c6dff940b7962766483b28d1008c073 into 05a16c2dc03c47138adb361e67f37241(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:51,891 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:51,891 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/A, priority=13, startTime=1732639251832; duration=0sec 2024-11-26T16:40:51,892 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:40:51,892 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:A 2024-11-26T16:40:51,892 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:51,893 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:51,894 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/C is initiating minor compaction (all files) 2024-11-26T16:40:51,894 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/C in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:51,894 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/2340fc53e7a84c70930207e945b8357c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/b44e0172af184bf8800b70fc3f3c9462, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/f834988a7d7b441cb01b807685a610fb] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=36.0 K 2024-11-26T16:40:51,895 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2340fc53e7a84c70930207e945b8357c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732639248392 2024-11-26T16:40:51,895 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting b44e0172af184bf8800b70fc3f3c9462, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732639249607 2024-11-26T16:40:51,895 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting f834988a7d7b441cb01b807685a610fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732639250257 2024-11-26T16:40:51,908 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#C#compaction#47 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:51,909 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/12ee76fc808343069ebd2dac990019ac is 50, key is test_row_0/C:col10/1732639250916/Put/seqid=0 2024-11-26T16:40:51,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741884_1060 (size=12663) 2024-11-26T16:40:51,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T16:40:51,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:51,959 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T16:40:51,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:40:51,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:51,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:40:51,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:51,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:40:51,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:51,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/da5124977fa946a0b0c5ecc1dc2923ce is 50, key is test_row_0/A:col10/1732639251020/Put/seqid=0 2024-11-26T16:40:51,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741885_1061 (size=12151) 2024-11-26T16:40:52,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T16:40:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:52,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:52,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639312205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:52,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:52,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639312207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:52,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:52,265 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/9522f6f6bb2f4c45abb016e514972c03 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/9522f6f6bb2f4c45abb016e514972c03 2024-11-26T16:40:52,274 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/B of 7c6dff940b7962766483b28d1008c073 into 9522f6f6bb2f4c45abb016e514972c03(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:52,274 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:52,274 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/B, priority=13, startTime=1732639251833; duration=0sec 2024-11-26T16:40:52,274 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:52,274 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:B 2024-11-26T16:40:52,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639312311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:52,327 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/12ee76fc808343069ebd2dac990019ac as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/12ee76fc808343069ebd2dac990019ac 2024-11-26T16:40:52,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639312321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:52,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:52,338 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/C of 7c6dff940b7962766483b28d1008c073 into 12ee76fc808343069ebd2dac990019ac(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:52,338 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:52,338 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/C, priority=13, startTime=1732639251833; duration=0sec 2024-11-26T16:40:52,338 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:52,338 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:C 2024-11-26T16:40:52,379 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/da5124977fa946a0b0c5ecc1dc2923ce 2024-11-26T16:40:52,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/1e2af255a7f2420aac06197257670e29 is 50, key is test_row_0/B:col10/1732639251020/Put/seqid=0 2024-11-26T16:40:52,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741886_1062 (size=12151) 2024-11-26T16:40:52,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639312521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:52,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:52,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639312531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:52,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T16:40:52,801 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/1e2af255a7f2420aac06197257670e29 2024-11-26T16:40:52,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52758 deadline: 1732639312801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:52,807 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_0' in 7c6dff940b7962766483b28d1008c073 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:40:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:52,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/f0dab98f97db4263af49185d1e0500f8 is 50, key is test_row_0/C:col10/1732639251020/Put/seqid=0 2024-11-26T16:40:52,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741887_1063 (size=12151) 2024-11-26T16:40:52,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52818 deadline: 1732639312831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:52,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639312831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:52,838 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in 7c6dff940b7962766483b28d1008c073 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:40:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:52,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52792 deadline: 1732639312842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:52,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639312842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:52,844 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_0' in 7c6dff940b7962766483b28d1008c073 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:40:52,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:52,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:53,227 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/f0dab98f97db4263af49185d1e0500f8 2024-11-26T16:40:53,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/da5124977fa946a0b0c5ecc1dc2923ce as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/da5124977fa946a0b0c5ecc1dc2923ce 2024-11-26T16:40:53,244 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/da5124977fa946a0b0c5ecc1dc2923ce, entries=150, sequenceid=227, filesize=11.9 K 2024-11-26T16:40:53,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/1e2af255a7f2420aac06197257670e29 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/1e2af255a7f2420aac06197257670e29 2024-11-26T16:40:53,258 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/1e2af255a7f2420aac06197257670e29, entries=150, sequenceid=227, filesize=11.9 K 2024-11-26T16:40:53,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/f0dab98f97db4263af49185d1e0500f8 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/f0dab98f97db4263af49185d1e0500f8 2024-11-26T16:40:53,271 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/f0dab98f97db4263af49185d1e0500f8, entries=150, sequenceid=227, filesize=11.9 K 2024-11-26T16:40:53,272 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 7c6dff940b7962766483b28d1008c073 in 1313ms, sequenceid=227, compaction requested=false 2024-11-26T16:40:53,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:53,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:53,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-26T16:40:53,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-26T16:40:53,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-26T16:40:53,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7780 sec 2024-11-26T16:40:53,281 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.7870 sec 2024-11-26T16:40:53,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:53,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T16:40:53,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:40:53,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:53,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:40:53,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:53,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:40:53,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:53,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/ef35824f21f34dc4a8665bafa94667b5 is 50, key is test_row_0/A:col10/1732639252204/Put/seqid=0 2024-11-26T16:40:53,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741888_1064 (size=14541) 2024-11-26T16:40:53,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/ef35824f21f34dc4a8665bafa94667b5 2024-11-26T16:40:53,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/e063e27cd0ef4ae781e85a3cf60bdfc0 is 50, key is test_row_0/B:col10/1732639252204/Put/seqid=0 2024-11-26T16:40:53,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741889_1065 (size=12151) 2024-11-26T16:40:53,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/e063e27cd0ef4ae781e85a3cf60bdfc0 2024-11-26T16:40:53,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/d9112c762eb343639178483409497075 is 50, key is test_row_0/C:col10/1732639252204/Put/seqid=0 2024-11-26T16:40:53,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:53,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639313401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:53,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:53,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:53,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:53,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:53,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639313402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:53,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741890_1066 (size=12151) 2024-11-26T16:40:53,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/d9112c762eb343639178483409497075 2024-11-26T16:40:53,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/ef35824f21f34dc4a8665bafa94667b5 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/ef35824f21f34dc4a8665bafa94667b5 2024-11-26T16:40:53,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/ef35824f21f34dc4a8665bafa94667b5, entries=200, sequenceid=245, filesize=14.2 K 2024-11-26T16:40:53,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/e063e27cd0ef4ae781e85a3cf60bdfc0 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/e063e27cd0ef4ae781e85a3cf60bdfc0 2024-11-26T16:40:53,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/e063e27cd0ef4ae781e85a3cf60bdfc0, entries=150, sequenceid=245, filesize=11.9 K 2024-11-26T16:40:53,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/d9112c762eb343639178483409497075 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/d9112c762eb343639178483409497075 2024-11-26T16:40:53,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/d9112c762eb343639178483409497075, entries=150, sequenceid=245, filesize=11.9 K 2024-11-26T16:40:53,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 7c6dff940b7962766483b28d1008c073 in 108ms, sequenceid=245, compaction requested=true 2024-11-26T16:40:53,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:53,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:40:53,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:53,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:40:53,462 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:53,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:53,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:40:53,462 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:53,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:40:53,464 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:53,464 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:53,464 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/B is initiating minor compaction (all files) 2024-11-26T16:40:53,464 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/A is initiating minor compaction (all files) 2024-11-26T16:40:53,464 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/B in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:53,464 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/A in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:53,464 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/05a16c2dc03c47138adb361e67f37241, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/da5124977fa946a0b0c5ecc1dc2923ce, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/ef35824f21f34dc4a8665bafa94667b5] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=38.4 K 2024-11-26T16:40:53,464 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/9522f6f6bb2f4c45abb016e514972c03, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/1e2af255a7f2420aac06197257670e29, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/e063e27cd0ef4ae781e85a3cf60bdfc0] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=36.1 K 2024-11-26T16:40:53,466 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 9522f6f6bb2f4c45abb016e514972c03, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732639250257 2024-11-26T16:40:53,466 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05a16c2dc03c47138adb361e67f37241, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732639250257 2024-11-26T16:40:53,467 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e2af255a7f2420aac06197257670e29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732639250990 2024-11-26T16:40:53,467 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting da5124977fa946a0b0c5ecc1dc2923ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732639250990 2024-11-26T16:40:53,468 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting e063e27cd0ef4ae781e85a3cf60bdfc0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732639252191 2024-11-26T16:40:53,468 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef35824f21f34dc4a8665bafa94667b5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732639252191 2024-11-26T16:40:53,483 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#B#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:53,484 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/895ca52adf134fff992fd088ae72f0e8 is 50, key is test_row_0/B:col10/1732639252204/Put/seqid=0 2024-11-26T16:40:53,488 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#A#compaction#55 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:53,489 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/61b6f787db084d89b6cfb50174073729 is 50, key is test_row_0/A:col10/1732639252204/Put/seqid=0 2024-11-26T16:40:53,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741891_1067 (size=12765) 2024-11-26T16:40:53,508 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/895ca52adf134fff992fd088ae72f0e8 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/895ca52adf134fff992fd088ae72f0e8 2024-11-26T16:40:53,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741892_1068 (size=12765) 2024-11-26T16:40:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:53,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T16:40:53,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:40:53,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:53,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:40:53,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:53,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:40:53,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:53,520 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/61b6f787db084d89b6cfb50174073729 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/61b6f787db084d89b6cfb50174073729 2024-11-26T16:40:53,530 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/B of 7c6dff940b7962766483b28d1008c073 into 895ca52adf134fff992fd088ae72f0e8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:53,530 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:53,530 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/B, priority=13, startTime=1732639253462; duration=0sec 2024-11-26T16:40:53,531 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:40:53,531 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:B 2024-11-26T16:40:53,531 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:53,532 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/A of 7c6dff940b7962766483b28d1008c073 into 61b6f787db084d89b6cfb50174073729(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:53,533 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:53,533 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/A, priority=13, startTime=1732639253461; duration=0sec 2024-11-26T16:40:53,533 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:53,533 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:A 2024-11-26T16:40:53,534 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:53,536 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/C is initiating minor compaction (all files) 2024-11-26T16:40:53,536 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/C in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:53,536 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/12ee76fc808343069ebd2dac990019ac, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/f0dab98f97db4263af49185d1e0500f8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/d9112c762eb343639178483409497075] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=36.1 K 2024-11-26T16:40:53,537 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 12ee76fc808343069ebd2dac990019ac, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732639250257 2024-11-26T16:40:53,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/2e2ff154098248e6817977e88c17bd78 is 50, key is test_row_0/A:col10/1732639253399/Put/seqid=0 2024-11-26T16:40:53,541 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting f0dab98f97db4263af49185d1e0500f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732639250990 2024-11-26T16:40:53,543 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting d9112c762eb343639178483409497075, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732639252191 2024-11-26T16:40:53,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741893_1069 (size=14691) 2024-11-26T16:40:53,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/2e2ff154098248e6817977e88c17bd78 2024-11-26T16:40:53,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:53,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639313561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:53,566 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#C#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:53,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:53,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:53,568 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/dca67aa262774aec9817597b21ddf875 is 50, key is test_row_0/C:col10/1732639252204/Put/seqid=0 2024-11-26T16:40:53,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:53,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639313563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:53,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:53,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:53,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:53,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/0a20b5332e02444ab530ee04ee310ac7 is 50, key is test_row_0/B:col10/1732639253399/Put/seqid=0 2024-11-26T16:40:53,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741894_1070 (size=12765) 2024-11-26T16:40:53,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741895_1071 (size=12301) 2024-11-26T16:40:53,592 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/dca67aa262774aec9817597b21ddf875 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/dca67aa262774aec9817597b21ddf875 2024-11-26T16:40:53,600 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/C of 7c6dff940b7962766483b28d1008c073 into dca67aa262774aec9817597b21ddf875(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:53,600 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:53,600 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/C, priority=13, startTime=1732639253462; duration=0sec 2024-11-26T16:40:53,600 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:53,600 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:C 2024-11-26T16:40:53,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T16:40:53,642 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:40:53,642 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:40:53,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-26T16:40:53,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T16:40:53,645 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:40:53,646 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:40:53,646 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:40:53,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639313671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:53,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:53,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:53,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:53,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639313671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:53,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:53,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T16:40:53,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T16:40:53,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:53,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:53,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:53,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:53,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:53,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:53,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639313881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:53,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:53,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639313881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:53,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T16:40:53,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:53,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:53,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:53,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:53,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T16:40:53,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/0a20b5332e02444ab530ee04ee310ac7 2024-11-26T16:40:53,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/4bfd473c4ee44e77a1c2b729f5260334 is 50, key is test_row_0/C:col10/1732639253399/Put/seqid=0 2024-11-26T16:40:54,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741896_1072 (size=12301) 2024-11-26T16:40:54,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/4bfd473c4ee44e77a1c2b729f5260334 2024-11-26T16:40:54,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/2e2ff154098248e6817977e88c17bd78 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2e2ff154098248e6817977e88c17bd78 2024-11-26T16:40:54,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2e2ff154098248e6817977e88c17bd78, entries=200, sequenceid=262, filesize=14.3 K 2024-11-26T16:40:54,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/0a20b5332e02444ab530ee04ee310ac7 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/0a20b5332e02444ab530ee04ee310ac7 2024-11-26T16:40:54,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/0a20b5332e02444ab530ee04ee310ac7, entries=150, sequenceid=262, filesize=12.0 K 2024-11-26T16:40:54,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/4bfd473c4ee44e77a1c2b729f5260334 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/4bfd473c4ee44e77a1c2b729f5260334 2024-11-26T16:40:54,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/4bfd473c4ee44e77a1c2b729f5260334, entries=150, sequenceid=262, filesize=12.0 K 2024-11-26T16:40:54,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 7c6dff940b7962766483b28d1008c073 in 517ms, sequenceid=262, compaction requested=false 2024-11-26T16:40:54,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:54,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T16:40:54,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:54,109 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T16:40:54,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:40:54,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:54,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:40:54,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:54,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:40:54,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:54,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/ce7bf6c118e548b7adf00fb9136e3fe0 is 50, key is test_row_0/A:col10/1732639253557/Put/seqid=0 2024-11-26T16:40:54,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741897_1073 (size=12301) 2024-11-26T16:40:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:54,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:54,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639314260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:54,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:54,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:54,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:54,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639314269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:54,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:54,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:54,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T16:40:54,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639314371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:54,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639314381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:54,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:54,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:54,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:54,526 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/ce7bf6c118e548b7adf00fb9136e3fe0 2024-11-26T16:40:54,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/3928da453abe4ecda31bdf159bb1d0db is 50, key is test_row_0/B:col10/1732639253557/Put/seqid=0 2024-11-26T16:40:54,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741898_1074 (size=12301) 2024-11-26T16:40:54,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639314581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:54,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:54,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:54,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:54,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639314591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:54,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:54,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:54,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T16:40:54,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639314891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:54,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639314901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:54,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:54,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:54,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:54,941 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/3928da453abe4ecda31bdf159bb1d0db 2024-11-26T16:40:54,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/a9f0a3fcf3b0458ab097002c455a0d3f is 50, key is test_row_0/C:col10/1732639253557/Put/seqid=0 2024-11-26T16:40:54,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741899_1075 (size=12301) 2024-11-26T16:40:55,367 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/a9f0a3fcf3b0458ab097002c455a0d3f 2024-11-26T16:40:55,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/ce7bf6c118e548b7adf00fb9136e3fe0 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/ce7bf6c118e548b7adf00fb9136e3fe0 2024-11-26T16:40:55,380 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/ce7bf6c118e548b7adf00fb9136e3fe0, entries=150, sequenceid=280, filesize=12.0 K 2024-11-26T16:40:55,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/3928da453abe4ecda31bdf159bb1d0db as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/3928da453abe4ecda31bdf159bb1d0db 2024-11-26T16:40:55,387 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/3928da453abe4ecda31bdf159bb1d0db, entries=150, sequenceid=280, filesize=12.0 K 2024-11-26T16:40:55,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/a9f0a3fcf3b0458ab097002c455a0d3f as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/a9f0a3fcf3b0458ab097002c455a0d3f 2024-11-26T16:40:55,393 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/a9f0a3fcf3b0458ab097002c455a0d3f, entries=150, sequenceid=280, filesize=12.0 K 2024-11-26T16:40:55,394 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 7c6dff940b7962766483b28d1008c073 in 1285ms, sequenceid=280, compaction requested=true 2024-11-26T16:40:55,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:55,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:55,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-26T16:40:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-26T16:40:55,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-26T16:40:55,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7500 sec 2024-11-26T16:40:55,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.7560 sec 2024-11-26T16:40:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:55,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T16:40:55,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:40:55,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:55,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:40:55,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:55,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:40:55,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:55,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/779241c50dba4376ac9ee334c8e29c72 is 50, key is test_row_0/A:col10/1732639254257/Put/seqid=0 2024-11-26T16:40:55,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741900_1076 (size=14741) 2024-11-26T16:40:55,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639315479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:55,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:55,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:55,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:55,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639315484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:55,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639315591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:55,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639315601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:55,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:55,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:55,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T16:40:55,792 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:40:55,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:40:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-26T16:40:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T16:40:55,795 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:40:55,796 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:40:55,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:40:55,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639315801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:55,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639315811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:55,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:55,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:55,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:55,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/779241c50dba4376ac9ee334c8e29c72 2024-11-26T16:40:55,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/fe9935789406457eab5932fdf9b40b4f is 50, key is test_row_0/B:col10/1732639254257/Put/seqid=0 2024-11-26T16:40:55,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741901_1077 (size=12301) 2024-11-26T16:40:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T16:40:55,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T16:40:55,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:55,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:55,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:55,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:55,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:56,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T16:40:56,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:56,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:56,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:56,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:56,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T16:40:56,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639316111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:56,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639316121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:56,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/fe9935789406457eab5932fdf9b40b4f 2024-11-26T16:40:56,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/35fed45635e849cd9b11a49ef3eec082 is 50, key is test_row_0/C:col10/1732639254257/Put/seqid=0 2024-11-26T16:40:56,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T16:40:56,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:56,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:56,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:56,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:56,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:56,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741902_1078 (size=12301) 2024-11-26T16:40:56,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/35fed45635e849cd9b11a49ef3eec082 2024-11-26T16:40:56,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/779241c50dba4376ac9ee334c8e29c72 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/779241c50dba4376ac9ee334c8e29c72 2024-11-26T16:40:56,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/779241c50dba4376ac9ee334c8e29c72, entries=200, sequenceid=298, filesize=14.4 K 2024-11-26T16:40:56,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/fe9935789406457eab5932fdf9b40b4f as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/fe9935789406457eab5932fdf9b40b4f 2024-11-26T16:40:56,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/fe9935789406457eab5932fdf9b40b4f, entries=150, sequenceid=298, filesize=12.0 K 2024-11-26T16:40:56,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/35fed45635e849cd9b11a49ef3eec082 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/35fed45635e849cd9b11a49ef3eec082 2024-11-26T16:40:56,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/35fed45635e849cd9b11a49ef3eec082, entries=150, sequenceid=298, filesize=12.0 K 2024-11-26T16:40:56,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 7c6dff940b7962766483b28d1008c073 in 911ms, sequenceid=298, compaction requested=true 2024-11-26T16:40:56,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:56,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:40:56,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:56,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:40:56,315 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:40:56,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:56,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:40:56,315 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:40:56,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:40:56,317 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54498 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:40:56,317 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/A is initiating minor compaction (all files) 2024-11-26T16:40:56,317 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/A in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:56,317 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:40:56,317 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/61b6f787db084d89b6cfb50174073729, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2e2ff154098248e6817977e88c17bd78, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/ce7bf6c118e548b7adf00fb9136e3fe0, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/779241c50dba4376ac9ee334c8e29c72] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=53.2 K 2024-11-26T16:40:56,317 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/B is initiating minor compaction (all files) 2024-11-26T16:40:56,318 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/B in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:56,318 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/895ca52adf134fff992fd088ae72f0e8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/0a20b5332e02444ab530ee04ee310ac7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/3928da453abe4ecda31bdf159bb1d0db, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/fe9935789406457eab5932fdf9b40b4f] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=48.5 K 2024-11-26T16:40:56,319 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61b6f787db084d89b6cfb50174073729, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732639252191 2024-11-26T16:40:56,319 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 895ca52adf134fff992fd088ae72f0e8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732639252191 2024-11-26T16:40:56,319 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e2ff154098248e6817977e88c17bd78, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732639253385 2024-11-26T16:40:56,320 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a20b5332e02444ab530ee04ee310ac7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732639253399 2024-11-26T16:40:56,320 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce7bf6c118e548b7adf00fb9136e3fe0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732639253546 2024-11-26T16:40:56,320 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 3928da453abe4ecda31bdf159bb1d0db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732639253546 2024-11-26T16:40:56,320 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 779241c50dba4376ac9ee334c8e29c72, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732639254250 2024-11-26T16:40:56,321 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting fe9935789406457eab5932fdf9b40b4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732639254250 2024-11-26T16:40:56,337 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#A#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:56,339 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/de44993be407454894db6c6acc87a5de is 50, key is test_row_0/A:col10/1732639254257/Put/seqid=0 2024-11-26T16:40:56,344 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#B#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:56,345 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/7503a7a06438456cb2567d02b45c35ce is 50, key is test_row_0/B:col10/1732639254257/Put/seqid=0 2024-11-26T16:40:56,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741903_1079 (size=13051) 2024-11-26T16:40:56,378 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/de44993be407454894db6c6acc87a5de as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/de44993be407454894db6c6acc87a5de 2024-11-26T16:40:56,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741904_1080 (size=13051) 2024-11-26T16:40:56,398 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c6dff940b7962766483b28d1008c073/A of 7c6dff940b7962766483b28d1008c073 into de44993be407454894db6c6acc87a5de(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:56,398 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:56,398 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/A, priority=12, startTime=1732639256315; duration=0sec 2024-11-26T16:40:56,398 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/7503a7a06438456cb2567d02b45c35ce as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/7503a7a06438456cb2567d02b45c35ce 2024-11-26T16:40:56,398 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:40:56,398 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:A 2024-11-26T16:40:56,399 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:40:56,407 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:40:56,407 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/C is initiating minor compaction (all files) 2024-11-26T16:40:56,407 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/C in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:56,408 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/dca67aa262774aec9817597b21ddf875, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/4bfd473c4ee44e77a1c2b729f5260334, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/a9f0a3fcf3b0458ab097002c455a0d3f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/35fed45635e849cd9b11a49ef3eec082] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=48.5 K 2024-11-26T16:40:56,412 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting dca67aa262774aec9817597b21ddf875, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732639252191 2024-11-26T16:40:56,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-26T16:40:56,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:56,414 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T16:40:56,414 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bfd473c4ee44e77a1c2b729f5260334, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732639253399 2024-11-26T16:40:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:40:56,415 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c6dff940b7962766483b28d1008c073/B of 7c6dff940b7962766483b28d1008c073 into 7503a7a06438456cb2567d02b45c35ce(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:56,415 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:56,415 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/B, priority=12, startTime=1732639256315; duration=0sec 2024-11-26T16:40:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:40:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:40:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:56,415 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:56,416 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:B 2024-11-26T16:40:56,418 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9f0a3fcf3b0458ab097002c455a0d3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732639253546 2024-11-26T16:40:56,419 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35fed45635e849cd9b11a49ef3eec082, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732639254250 2024-11-26T16:40:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T16:40:56,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/0b4c866f9bf34527ae58fd7ba21cc13a is 50, key is test_row_0/A:col10/1732639255472/Put/seqid=0 2024-11-26T16:40:56,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741905_1081 (size=12301) 2024-11-26T16:40:56,446 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/0b4c866f9bf34527ae58fd7ba21cc13a 2024-11-26T16:40:56,457 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#C#compaction#69 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:56,458 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/36d97fec48194ed49e35cae445975adc is 50, key is test_row_0/C:col10/1732639254257/Put/seqid=0 2024-11-26T16:40:56,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/fbf18f618e1f4ae891076ef2700fd9a3 is 50, key is test_row_0/B:col10/1732639255472/Put/seqid=0 2024-11-26T16:40:56,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741906_1082 (size=13051) 2024-11-26T16:40:56,473 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/36d97fec48194ed49e35cae445975adc as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/36d97fec48194ed49e35cae445975adc 2024-11-26T16:40:56,493 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c6dff940b7962766483b28d1008c073/C of 7c6dff940b7962766483b28d1008c073 into 36d97fec48194ed49e35cae445975adc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:56,493 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:56,493 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/C, priority=12, startTime=1732639256315; duration=0sec 2024-11-26T16:40:56,494 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:56,494 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:C 2024-11-26T16:40:56,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741907_1083 (size=12301) 2024-11-26T16:40:56,496 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/fbf18f618e1f4ae891076ef2700fd9a3 2024-11-26T16:40:56,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/5431feab617c4b39843323bc95199572 is 50, key is test_row_0/C:col10/1732639255472/Put/seqid=0 2024-11-26T16:40:56,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741908_1084 (size=12301) 2024-11-26T16:40:56,528 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/5431feab617c4b39843323bc95199572 2024-11-26T16:40:56,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/0b4c866f9bf34527ae58fd7ba21cc13a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/0b4c866f9bf34527ae58fd7ba21cc13a 2024-11-26T16:40:56,547 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/0b4c866f9bf34527ae58fd7ba21cc13a, entries=150, sequenceid=314, filesize=12.0 K 2024-11-26T16:40:56,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/fbf18f618e1f4ae891076ef2700fd9a3 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/fbf18f618e1f4ae891076ef2700fd9a3 2024-11-26T16:40:56,556 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/fbf18f618e1f4ae891076ef2700fd9a3, entries=150, sequenceid=314, filesize=12.0 K 2024-11-26T16:40:56,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/5431feab617c4b39843323bc95199572 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/5431feab617c4b39843323bc95199572 2024-11-26T16:40:56,571 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/5431feab617c4b39843323bc95199572, entries=150, sequenceid=314, filesize=12.0 K 2024-11-26T16:40:56,578 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 7c6dff940b7962766483b28d1008c073 in 164ms, sequenceid=314, compaction requested=false 2024-11-26T16:40:56,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:56,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:56,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-26T16:40:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-26T16:40:56,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-26T16:40:56,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 785 msec 2024-11-26T16:40:56,592 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 798 msec 2024-11-26T16:40:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:56,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T16:40:56,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:40:56,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:56,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:40:56,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:56,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:40:56,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:56,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/67662f8fe9574632859f3695ebcd9164 is 50, key is test_row_0/A:col10/1732639256650/Put/seqid=0 2024-11-26T16:40:56,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741909_1085 (size=12301) 2024-11-26T16:40:56,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732639256758, max timestamp=1732639256778Num uniques 859; 2024-11-26T16:40:56,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:40:56,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639256758, max timestamp=1732639256778Num uniques 859; 2024-11-26T16:40:56,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:40:56,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639256758, max timestamp=1732639256791Num uniques 859; 2024-11-26T16:40:56,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:40:56,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:56,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639316880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:56,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639316885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:56,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:56,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:56,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-26T16:40:56,931 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:40:56,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:40:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-26T16:40:56,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T16:40:56,940 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:40:56,941 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:40:56,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:40:56,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639316991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:56,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:56,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:56,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:57,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639317001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:57,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:57,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:57,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T16:40:57,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T16:40:57,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:57,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:57,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:57,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:57,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:57,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/67662f8fe9574632859f3695ebcd9164 2024-11-26T16:40:57,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/8b4a56bd7b664b6dba232f2b4c92ac77 is 50, key is test_row_0/B:col10/1732639256650/Put/seqid=0 2024-11-26T16:40:57,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741910_1086 (size=12301) 2024-11-26T16:40:57,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639317212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:57,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:57,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:57,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639317212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:57,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:57,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:57,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:57,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T16:40:57,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:57,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:57,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:57,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T16:40:57,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:57,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T16:40:57,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:57,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:57,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:57,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:57,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:57,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639317521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:57,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:57,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:57,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:57,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639317522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:57,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:57,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:57,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:57,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T16:40:57,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:57,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T16:40:57,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:57,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:57,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:40:57,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/8b4a56bd7b664b6dba232f2b4c92ac77 2024-11-26T16:40:57,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/8029d31c8c7049ee87dc0233aeb1c371 is 50, key is test_row_0/C:col10/1732639256650/Put/seqid=0 2024-11-26T16:40:57,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741911_1087 (size=12301) 2024-11-26T16:40:57,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/8029d31c8c7049ee87dc0233aeb1c371 2024-11-26T16:40:57,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/67662f8fe9574632859f3695ebcd9164 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/67662f8fe9574632859f3695ebcd9164 2024-11-26T16:40:57,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/67662f8fe9574632859f3695ebcd9164, entries=150, sequenceid=326, filesize=12.0 K 2024-11-26T16:40:57,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/8b4a56bd7b664b6dba232f2b4c92ac77 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/8b4a56bd7b664b6dba232f2b4c92ac77 2024-11-26T16:40:57,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/8b4a56bd7b664b6dba232f2b4c92ac77, entries=150, sequenceid=326, filesize=12.0 K 2024-11-26T16:40:57,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/8029d31c8c7049ee87dc0233aeb1c371 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/8029d31c8c7049ee87dc0233aeb1c371 2024-11-26T16:40:57,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/8029d31c8c7049ee87dc0233aeb1c371, entries=150, sequenceid=326, filesize=12.0 K 2024-11-26T16:40:57,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for 7c6dff940b7962766483b28d1008c073 in 1000ms, sequenceid=326, compaction requested=true 2024-11-26T16:40:57,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:57,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:40:57,670 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:57,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:57,670 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:57,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:40:57,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:57,672 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:57,672 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:57,672 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/A is initiating minor compaction (all files) 2024-11-26T16:40:57,672 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/B is initiating minor compaction (all files) 2024-11-26T16:40:57,672 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/A in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:57,672 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/B in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:57,672 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/de44993be407454894db6c6acc87a5de, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/0b4c866f9bf34527ae58fd7ba21cc13a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/67662f8fe9574632859f3695ebcd9164] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=36.8 K 2024-11-26T16:40:57,672 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/7503a7a06438456cb2567d02b45c35ce, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/fbf18f618e1f4ae891076ef2700fd9a3, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/8b4a56bd7b664b6dba232f2b4c92ac77] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=36.8 K 2024-11-26T16:40:57,673 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting de44993be407454894db6c6acc87a5de, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732639254250 2024-11-26T16:40:57,673 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 7503a7a06438456cb2567d02b45c35ce, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732639254250 2024-11-26T16:40:57,673 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b4c866f9bf34527ae58fd7ba21cc13a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732639255472 2024-11-26T16:40:57,673 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting fbf18f618e1f4ae891076ef2700fd9a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732639255472 2024-11-26T16:40:57,674 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67662f8fe9574632859f3695ebcd9164, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732639256627 2024-11-26T16:40:57,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:40:57,674 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b4a56bd7b664b6dba232f2b4c92ac77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732639256627 2024-11-26T16:40:57,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:40:57,714 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#A#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:57,715 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/e1d5c816299846d2b81c2bae65fb39c0 is 50, key is test_row_0/A:col10/1732639256650/Put/seqid=0 2024-11-26T16:40:57,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-26T16:40:57,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:57,717 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=129.71 KB heapSize=370.14 KB 2024-11-26T16:40:57,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:40:57,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:40:57,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:40:57,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:40:57,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:40:57,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:40:57,726 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#B#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:57,727 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/d7f4065d76d54a629e21aed53c4be20f is 50, key is test_row_0/B:col10/1732639256650/Put/seqid=0 2024-11-26T16:40:57,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/dd89362028eb4ff7bb19eb714fb6a557 is 50, key is test_row_0/A:col10/1732639256875/Put/seqid=0 2024-11-26T16:40:57,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741912_1088 (size=13153) 2024-11-26T16:40:57,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741913_1089 (size=13153) 2024-11-26T16:40:57,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741914_1090 (size=12301) 2024-11-26T16:40:58,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:58,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:58,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T16:40:58,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:58,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639318072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:58,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:58,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:58,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:58,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639318073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:58,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:58,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:58,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:58,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:58,172 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/e1d5c816299846d2b81c2bae65fb39c0 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/e1d5c816299846d2b81c2bae65fb39c0 2024-11-26T16:40:58,195 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/A of 7c6dff940b7962766483b28d1008c073 into e1d5c816299846d2b81c2bae65fb39c0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:58,195 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:58,195 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/A, priority=13, startTime=1732639257670; duration=0sec 2024-11-26T16:40:58,195 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:40:58,195 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:A 2024-11-26T16:40:58,196 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:40:58,197 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:40:58,197 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/C is initiating minor compaction (all files) 2024-11-26T16:40:58,197 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/C in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:58,197 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/36d97fec48194ed49e35cae445975adc, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/5431feab617c4b39843323bc95199572, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/8029d31c8c7049ee87dc0233aeb1c371] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=36.8 K 2024-11-26T16:40:58,198 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36d97fec48194ed49e35cae445975adc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732639254250 2024-11-26T16:40:58,198 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5431feab617c4b39843323bc95199572, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732639255472 2024-11-26T16:40:58,199 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8029d31c8c7049ee87dc0233aeb1c371, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732639256627 2024-11-26T16:40:58,209 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/d7f4065d76d54a629e21aed53c4be20f as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/d7f4065d76d54a629e21aed53c4be20f 2024-11-26T16:40:58,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:58,213 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#C#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:40:58,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639318201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:58,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:58,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:58,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:58,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:58,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639318201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:58,215 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/1e02d551d46c4043b474e9ce803546f5 is 50, key is test_row_0/C:col10/1732639256650/Put/seqid=0 2024-11-26T16:40:58,221 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/B of 7c6dff940b7962766483b28d1008c073 into d7f4065d76d54a629e21aed53c4be20f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:58,221 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:58,221 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/B, priority=13, startTime=1732639257670; duration=0sec 2024-11-26T16:40:58,221 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:58,221 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:B 2024-11-26T16:40:58,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741915_1091 (size=13153) 2024-11-26T16:40:58,239 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/dd89362028eb4ff7bb19eb714fb6a557 2024-11-26T16:40:58,247 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/1e02d551d46c4043b474e9ce803546f5 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/1e02d551d46c4043b474e9ce803546f5 2024-11-26T16:40:58,256 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/C of 7c6dff940b7962766483b28d1008c073 into 1e02d551d46c4043b474e9ce803546f5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:40:58,256 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:58,256 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/C, priority=13, startTime=1732639257672; duration=0sec 2024-11-26T16:40:58,256 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:40:58,256 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:C 2024-11-26T16:40:58,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/3bf74dd21ed34ec785d0c7745197a470 is 50, key is test_row_0/B:col10/1732639256875/Put/seqid=0 2024-11-26T16:40:58,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741916_1092 (size=12301) 2024-11-26T16:40:58,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639318421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:58,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639318422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:58,676 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/3bf74dd21ed34ec785d0c7745197a470 2024-11-26T16:40:58,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/9d00fb94b51a4b13bf70226de663795d is 50, key is test_row_0/C:col10/1732639256875/Put/seqid=0 2024-11-26T16:40:58,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741917_1093 (size=12301) 2024-11-26T16:40:58,740 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/9d00fb94b51a4b13bf70226de663795d 2024-11-26T16:40:58,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639318741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:58,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:58,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:58,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:58,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639318743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:58,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:58,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:58,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:58,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/dd89362028eb4ff7bb19eb714fb6a557 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/dd89362028eb4ff7bb19eb714fb6a557 2024-11-26T16:40:58,756 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/dd89362028eb4ff7bb19eb714fb6a557, entries=150, sequenceid=362, filesize=12.0 K 2024-11-26T16:40:58,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/3bf74dd21ed34ec785d0c7745197a470 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/3bf74dd21ed34ec785d0c7745197a470 2024-11-26T16:40:58,794 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/3bf74dd21ed34ec785d0c7745197a470, entries=150, sequenceid=362, filesize=12.0 K 2024-11-26T16:40:58,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/9d00fb94b51a4b13bf70226de663795d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/9d00fb94b51a4b13bf70226de663795d 2024-11-26T16:40:58,807 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/9d00fb94b51a4b13bf70226de663795d, entries=150, sequenceid=362, filesize=12.0 K 2024-11-26T16:40:58,808 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~370.09 KB/378976, currentSize=53.67 KB/54960 for 7c6dff940b7962766483b28d1008c073 in 1092ms, sequenceid=362, compaction requested=false 2024-11-26T16:40:58,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:58,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:58,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-26T16:40:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-26T16:40:58,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-26T16:40:58,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8690 sec 2024-11-26T16:40:58,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.8820 sec 2024-11-26T16:40:59,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-26T16:40:59,082 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:40:59,083 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:40:59,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-26T16:40:59,089 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:40:59,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T16:40:59,093 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:40:59,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:40:59,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T16:40:59,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-26T16:40:59,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:59,253 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T16:40:59,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:40:59,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:59,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:40:59,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:59,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:40:59,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:40:59,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:40:59,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/b91293270ce345fbb8e2300a347a447c is 50, key is test_row_0/A:col10/1732639258069/Put/seqid=0 2024-11-26T16:40:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:40:59,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741918_1094 (size=9857) 2024-11-26T16:40:59,288 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/b91293270ce345fbb8e2300a347a447c 2024-11-26T16:40:59,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/65aa5fcfffd24e6995d1c02d454a91d5 is 50, key is test_row_0/B:col10/1732639258069/Put/seqid=0 2024-11-26T16:40:59,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741919_1095 (size=9857) 2024-11-26T16:40:59,373 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/65aa5fcfffd24e6995d1c02d454a91d5 2024-11-26T16:40:59,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639259341, max timestamp=1732639259373Num uniques 859; 2024-11-26T16:40:59,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639259341, max timestamp=1732639259373Num uniques 859; 2024-11-26T16:40:59,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:40:59,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:40:59,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639259341, max timestamp=1732639259355Num uniques 859; 2024-11-26T16:40:59,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:40:59,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/7bd505b89e5e4e99a7f6647eeb81239c is 50, key is test_row_0/C:col10/1732639258069/Put/seqid=0 2024-11-26T16:40:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T16:40:59,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741920_1096 (size=9857) 2024-11-26T16:40:59,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639319469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:59,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639319471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:59,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:59,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639319581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:59,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639319582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:59,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:59,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:59,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:59,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T16:40:59,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639319801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:59,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:40:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:59,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639319802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:40:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:40:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:40:59,849 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/7bd505b89e5e4e99a7f6647eeb81239c 2024-11-26T16:40:59,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/b91293270ce345fbb8e2300a347a447c as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/b91293270ce345fbb8e2300a347a447c 2024-11-26T16:40:59,878 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/b91293270ce345fbb8e2300a347a447c, entries=100, sequenceid=376, filesize=9.6 K 2024-11-26T16:40:59,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/65aa5fcfffd24e6995d1c02d454a91d5 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/65aa5fcfffd24e6995d1c02d454a91d5 2024-11-26T16:40:59,899 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/65aa5fcfffd24e6995d1c02d454a91d5, entries=100, sequenceid=376, filesize=9.6 K 2024-11-26T16:40:59,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/7bd505b89e5e4e99a7f6647eeb81239c as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/7bd505b89e5e4e99a7f6647eeb81239c 2024-11-26T16:40:59,912 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/7bd505b89e5e4e99a7f6647eeb81239c, entries=100, sequenceid=376, filesize=9.6 K 2024-11-26T16:40:59,917 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 7c6dff940b7962766483b28d1008c073 in 664ms, sequenceid=376, compaction requested=true 2024-11-26T16:40:59,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:40:59,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:40:59,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-26T16:40:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-26T16:40:59,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-26T16:40:59,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 821 msec 2024-11-26T16:40:59,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 842 msec 2024-11-26T16:41:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:41:00,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-26T16:41:00,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:41:00,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:41:00,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:41:00,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:41:00,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:41:00,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:41:00,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/2da1c3203da0465f8c64d0ee328ad612 is 50, key is test_row_0/A:col10/1732639259469/Put/seqid=0 2024-11-26T16:41:00,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639320165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:00,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639320166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:00,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741921_1097 (size=12301) 2024-11-26T16:41:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-26T16:41:00,231 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:00,232 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:41:00,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-26T16:41:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T16:41:00,238 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:41:00,240 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:41:00,240 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:41:00,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:00,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639320282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:00,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:00,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639320284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T16:41:00,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T16:41:00,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:00,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:41:00,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:00,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:00,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:00,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:00,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639320492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:00,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:00,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:00,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:00,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639320491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:00,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T16:41:00,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:00,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:41:00,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:00,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:00,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T16:41:00,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:00,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/2da1c3203da0465f8c64d0ee328ad612 2024-11-26T16:41:00,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/4815a6fe89ef48ceb8c52b86732721c0 is 50, key is test_row_0/B:col10/1732639259469/Put/seqid=0 2024-11-26T16:41:00,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741922_1098 (size=12301) 2024-11-26T16:41:00,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/4815a6fe89ef48ceb8c52b86732721c0 2024-11-26T16:41:00,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/da9d5f17761347d4a43f0799ee0f7660 is 50, key is test_row_0/C:col10/1732639259469/Put/seqid=0 2024-11-26T16:41:00,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741923_1099 (size=12301) 2024-11-26T16:41:00,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T16:41:00,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:00,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:41:00,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:00,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:00,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:00,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639320811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:00,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:00,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:00,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639320822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:00,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T16:41:00,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:00,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:41:00,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:00,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:00,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:00,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T16:41:01,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T16:41:01,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:01,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:41:01,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:01,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:01,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:01,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/da9d5f17761347d4a43f0799ee0f7660 2024-11-26T16:41:01,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/2da1c3203da0465f8c64d0ee328ad612 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2da1c3203da0465f8c64d0ee328ad612 2024-11-26T16:41:01,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2da1c3203da0465f8c64d0ee328ad612, entries=150, sequenceid=414, filesize=12.0 K 2024-11-26T16:41:01,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/4815a6fe89ef48ceb8c52b86732721c0 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/4815a6fe89ef48ceb8c52b86732721c0 2024-11-26T16:41:01,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/4815a6fe89ef48ceb8c52b86732721c0, entries=150, sequenceid=414, filesize=12.0 K 2024-11-26T16:41:01,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/da9d5f17761347d4a43f0799ee0f7660 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/da9d5f17761347d4a43f0799ee0f7660 2024-11-26T16:41:01,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/da9d5f17761347d4a43f0799ee0f7660, entries=150, sequenceid=414, filesize=12.0 K 2024-11-26T16:41:01,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 7c6dff940b7962766483b28d1008c073 in 970ms, sequenceid=414, compaction requested=true 2024-11-26T16:41:01,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:41:01,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:41:01,095 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:41:01,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:01,095 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:41:01,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:41:01,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:01,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:41:01,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:01,101 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47612 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:41:01,101 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47612 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:41:01,101 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/A is initiating minor compaction (all files) 2024-11-26T16:41:01,101 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/B is initiating minor compaction (all files) 2024-11-26T16:41:01,101 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/A in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:01,101 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/B in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:01,102 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/d7f4065d76d54a629e21aed53c4be20f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/3bf74dd21ed34ec785d0c7745197a470, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/65aa5fcfffd24e6995d1c02d454a91d5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/4815a6fe89ef48ceb8c52b86732721c0] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=46.5 K 2024-11-26T16:41:01,102 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/e1d5c816299846d2b81c2bae65fb39c0, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/dd89362028eb4ff7bb19eb714fb6a557, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/b91293270ce345fbb8e2300a347a447c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2da1c3203da0465f8c64d0ee328ad612] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=46.5 K 2024-11-26T16:41:01,102 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting d7f4065d76d54a629e21aed53c4be20f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732639256627 2024-11-26T16:41:01,102 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1d5c816299846d2b81c2bae65fb39c0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732639256627 2024-11-26T16:41:01,103 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd89362028eb4ff7bb19eb714fb6a557, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732639256875 2024-11-26T16:41:01,103 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bf74dd21ed34ec785d0c7745197a470, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732639256875 2024-11-26T16:41:01,103 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting b91293270ce345fbb8e2300a347a447c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732639258069 2024-11-26T16:41:01,103 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 65aa5fcfffd24e6995d1c02d454a91d5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732639258069 2024-11-26T16:41:01,104 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2da1c3203da0465f8c64d0ee328ad612, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732639259445 2024-11-26T16:41:01,104 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 4815a6fe89ef48ceb8c52b86732721c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732639259445 2024-11-26T16:41:01,121 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#B#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:01,122 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/9279378fde12495fb84b892708b195d4 is 50, key is test_row_0/B:col10/1732639259469/Put/seqid=0 2024-11-26T16:41:01,129 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#A#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:01,130 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/27773d4cb7224737b32025ac011433a7 is 50, key is test_row_0/A:col10/1732639259469/Put/seqid=0 2024-11-26T16:41:01,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741925_1101 (size=13289) 2024-11-26T16:41:01,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741924_1100 (size=13289) 2024-11-26T16:41:01,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-26T16:41:01,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:01,169 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-26T16:41:01,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:41:01,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:01,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:41:01,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:01,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:41:01,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:01,172 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/9279378fde12495fb84b892708b195d4 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/9279378fde12495fb84b892708b195d4 2024-11-26T16:41:01,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/7ef8403d3f12435cad321028b995b89f is 50, key is test_row_0/A:col10/1732639260164/Put/seqid=0 2024-11-26T16:41:01,194 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c6dff940b7962766483b28d1008c073/B of 7c6dff940b7962766483b28d1008c073 into 9279378fde12495fb84b892708b195d4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:01,194 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:41:01,194 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/B, priority=12, startTime=1732639261095; duration=0sec 2024-11-26T16:41:01,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741926_1102 (size=12301) 2024-11-26T16:41:01,196 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/7ef8403d3f12435cad321028b995b89f 2024-11-26T16:41:01,197 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:01,197 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:B 2024-11-26T16:41:01,198 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:41:01,202 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47612 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:41:01,202 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/C is initiating minor compaction (all files) 2024-11-26T16:41:01,202 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/C in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:01,203 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/1e02d551d46c4043b474e9ce803546f5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/9d00fb94b51a4b13bf70226de663795d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/7bd505b89e5e4e99a7f6647eeb81239c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/da9d5f17761347d4a43f0799ee0f7660] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=46.5 K 2024-11-26T16:41:01,208 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e02d551d46c4043b474e9ce803546f5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732639256627 2024-11-26T16:41:01,208 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d00fb94b51a4b13bf70226de663795d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732639256875 2024-11-26T16:41:01,209 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bd505b89e5e4e99a7f6647eeb81239c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732639258069 2024-11-26T16:41:01,210 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting da9d5f17761347d4a43f0799ee0f7660, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732639259445 2024-11-26T16:41:01,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/24e06f353b2c4a35a926bae23d3157f1 is 50, key is test_row_0/B:col10/1732639260164/Put/seqid=0 2024-11-26T16:41:01,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741927_1103 (size=12301) 2024-11-26T16:41:01,225 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/24e06f353b2c4a35a926bae23d3157f1 2024-11-26T16:41:01,226 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#C#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:01,227 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/f3f2c76648cd4089b3ae17a6f6ebb544 is 50, key is test_row_0/C:col10/1732639259469/Put/seqid=0 2024-11-26T16:41:01,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741928_1104 (size=13289) 2024-11-26T16:41:01,245 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/f3f2c76648cd4089b3ae17a6f6ebb544 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/f3f2c76648cd4089b3ae17a6f6ebb544 2024-11-26T16:41:01,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/e9d0f863c6804ce9815d9835d8af1cfd is 50, key is test_row_0/C:col10/1732639260164/Put/seqid=0 2024-11-26T16:41:01,254 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c6dff940b7962766483b28d1008c073/C of 7c6dff940b7962766483b28d1008c073 into f3f2c76648cd4089b3ae17a6f6ebb544(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:01,254 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:41:01,254 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/C, priority=12, startTime=1732639261096; duration=0sec 2024-11-26T16:41:01,254 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:01,254 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:C 2024-11-26T16:41:01,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741929_1105 (size=12301) 2024-11-26T16:41:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:41:01,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:41:01,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T16:41:01,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732639261374, max timestamp=1732639261400Num uniques 859; 2024-11-26T16:41:01,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732639261374, max timestamp=1732639261400Num uniques 859; 2024-11-26T16:41:01,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:41:01,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:41:01,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732639261374, max timestamp=1732639261400Num uniques 859; 2024-11-26T16:41:01,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:41:01,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732639261467, max timestamp=1732639261510Num uniques 1068; 2024-11-26T16:41:01,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732639261467, max timestamp=1732639261510Num uniques 1068; 2024-11-26T16:41:01,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-26T16:41:01,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732639261467, max timestamp=1732639261510Num uniques 1068; 2024-11-26T16:41:01,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-26T16:41:01,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-26T16:41:01,562 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/27773d4cb7224737b32025ac011433a7 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/27773d4cb7224737b32025ac011433a7 2024-11-26T16:41:01,570 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7c6dff940b7962766483b28d1008c073/A of 7c6dff940b7962766483b28d1008c073 into 27773d4cb7224737b32025ac011433a7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:01,570 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:41:01,570 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/A, priority=12, startTime=1732639261095; duration=0sec 2024-11-26T16:41:01,570 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:01,570 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:A 2024-11-26T16:41:01,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639321625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:01,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:01,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:01,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639321628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:01,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:01,663 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/e9d0f863c6804ce9815d9835d8af1cfd 2024-11-26T16:41:01,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/7ef8403d3f12435cad321028b995b89f as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/7ef8403d3f12435cad321028b995b89f 2024-11-26T16:41:01,683 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/7ef8403d3f12435cad321028b995b89f, entries=150, sequenceid=423, filesize=12.0 K 2024-11-26T16:41:01,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/24e06f353b2c4a35a926bae23d3157f1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/24e06f353b2c4a35a926bae23d3157f1 2024-11-26T16:41:01,695 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/24e06f353b2c4a35a926bae23d3157f1, entries=150, sequenceid=423, filesize=12.0 K 2024-11-26T16:41:01,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/e9d0f863c6804ce9815d9835d8af1cfd as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/e9d0f863c6804ce9815d9835d8af1cfd 2024-11-26T16:41:01,708 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/e9d0f863c6804ce9815d9835d8af1cfd, entries=150, sequenceid=423, filesize=12.0 K 2024-11-26T16:41:01,710 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 7c6dff940b7962766483b28d1008c073 in 541ms, sequenceid=423, compaction requested=false 2024-11-26T16:41:01,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:41:01,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:01,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-26T16:41:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-26T16:41:01,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-26T16:41:01,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4720 sec 2024-11-26T16:41:01,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.4940 sec 2024-11-26T16:41:01,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732639261572, max timestamp=1732639261744Num uniques 1077; 2024-11-26T16:41:01,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-26T16:41:01,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732639261572, max timestamp=1732639261744Num uniques 1077; 2024-11-26T16:41:01,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732639261572, max timestamp=1732639261744Num uniques 1077; 2024-11-26T16:41:01,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-26T16:41:01,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-26T16:41:01,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=53.67 KB heapSize=144.23 KB 2024-11-26T16:41:01,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:41:01,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:41:01,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:41:01,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:41:01,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:41:01,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:41:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:41:01,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/69bff1aa616a4b62ae57682a9d2fe3f5 is 50, key is test_row_0/A:col10/1732639261744/Put/seqid=0 2024-11-26T16:41:01,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741930_1106 (size=12301) 2024-11-26T16:41:01,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732639261827, max timestamp=1732639261844Num uniques 859; 2024-11-26T16:41:01,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639261827, max timestamp=1732639261844Num uniques 859; 2024-11-26T16:41:01,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:41:01,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:41:01,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639261827, max timestamp=1732639261844Num uniques 859; 2024-11-26T16:41:01,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:41:01,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639321918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:01,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:01,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:01,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:01,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639321920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:01,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:01,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:01,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:02,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639322032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:02,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:02,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639322032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:02,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:02,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:02,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/69bff1aa616a4b62ae57682a9d2fe3f5 2024-11-26T16:41:02,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/c019e9b6d12b4861b67b4c5e6af913b7 is 50, key is test_row_0/B:col10/1732639261744/Put/seqid=0 2024-11-26T16:41:02,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741931_1107 (size=12301) 2024-11-26T16:41:02,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/c019e9b6d12b4861b67b4c5e6af913b7 2024-11-26T16:41:02,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/3eb8d2c296ce4c55ac8e83c1a669f54d is 50, key is test_row_0/C:col10/1732639261744/Put/seqid=0 2024-11-26T16:41:02,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741932_1108 (size=12301) 2024-11-26T16:41:02,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/3eb8d2c296ce4c55ac8e83c1a669f54d 2024-11-26T16:41:02,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/69bff1aa616a4b62ae57682a9d2fe3f5 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/69bff1aa616a4b62ae57682a9d2fe3f5 2024-11-26T16:41:02,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/69bff1aa616a4b62ae57682a9d2fe3f5, entries=150, sequenceid=485, filesize=12.0 K 2024-11-26T16:41:02,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/c019e9b6d12b4861b67b4c5e6af913b7 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/c019e9b6d12b4861b67b4c5e6af913b7 2024-11-26T16:41:02,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639322241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:02,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639322241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:02,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/c019e9b6d12b4861b67b4c5e6af913b7, entries=150, sequenceid=485, filesize=12.0 K 2024-11-26T16:41:02,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/3eb8d2c296ce4c55ac8e83c1a669f54d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/3eb8d2c296ce4c55ac8e83c1a669f54d 2024-11-26T16:41:02,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/3eb8d2c296ce4c55ac8e83c1a669f54d, entries=150, sequenceid=485, filesize=12.0 K 2024-11-26T16:41:02,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~144.19 KB/147648, currentSize=129.71 KB/132820 for 7c6dff940b7962766483b28d1008c073 in 501ms, sequenceid=485, compaction requested=true 2024-11-26T16:41:02,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:41:02,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:41:02,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:02,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:41:02,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:02,264 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:02,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:41:02,264 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:02,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:02,266 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:02,266 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/A is initiating minor compaction (all files) 2024-11-26T16:41:02,266 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/A in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:02,266 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/27773d4cb7224737b32025ac011433a7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/7ef8403d3f12435cad321028b995b89f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/69bff1aa616a4b62ae57682a9d2fe3f5] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=37.0 K 2024-11-26T16:41:02,267 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:02,267 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/B is initiating minor compaction (all files) 2024-11-26T16:41:02,267 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27773d4cb7224737b32025ac011433a7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732639259445 2024-11-26T16:41:02,267 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/B in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:02,267 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/9279378fde12495fb84b892708b195d4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/24e06f353b2c4a35a926bae23d3157f1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/c019e9b6d12b4861b67b4c5e6af913b7] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=37.0 K 2024-11-26T16:41:02,267 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ef8403d3f12435cad321028b995b89f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1732639260150 2024-11-26T16:41:02,268 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69bff1aa616a4b62ae57682a9d2fe3f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1732639261744 2024-11-26T16:41:02,268 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 9279378fde12495fb84b892708b195d4, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732639259445 2024-11-26T16:41:02,269 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 24e06f353b2c4a35a926bae23d3157f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1732639260150 2024-11-26T16:41:02,269 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting c019e9b6d12b4861b67b4c5e6af913b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1732639261744 2024-11-26T16:41:02,278 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#A#compaction#96 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:02,278 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/52b6d06714bd4469a970fb5f00fa22c2 is 50, key is test_row_0/A:col10/1732639261744/Put/seqid=0 2024-11-26T16:41:02,281 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#B#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:02,281 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/107642cce0d4407b8d1083cafe273cbe is 50, key is test_row_0/B:col10/1732639261744/Put/seqid=0 2024-11-26T16:41:02,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741934_1110 (size=13391) 2024-11-26T16:41:02,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741933_1109 (size=13391) 2024-11-26T16:41:02,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-26T16:41:02,381 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:02,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:41:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-26T16:41:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T16:41:02,385 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:41:02,388 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:41:02,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:41:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T16:41:02,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-26T16:41:02,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:02,544 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=129.71 KB heapSize=370.14 KB 2024-11-26T16:41:02,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:41:02,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:41:02,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:41:02,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:41:02,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:41:02,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:41:02,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/7bc5f9a906a744a9a53ab82da4cba063 is 50, key is test_row_0/A:col10/1732639261868/Put/seqid=0 2024-11-26T16:41:02,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741935_1111 (size=12301) 2024-11-26T16:41:02,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:41:02,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. as already flushing 2024-11-26T16:41:02,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639322590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:02,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639322603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T16:41:02,705 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/107642cce0d4407b8d1083cafe273cbe as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/107642cce0d4407b8d1083cafe273cbe 2024-11-26T16:41:02,706 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/52b6d06714bd4469a970fb5f00fa22c2 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/52b6d06714bd4469a970fb5f00fa22c2 2024-11-26T16:41:02,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639322712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639322712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:02,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:02,716 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/A of 7c6dff940b7962766483b28d1008c073 into 52b6d06714bd4469a970fb5f00fa22c2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:02,717 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:41:02,717 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/B of 7c6dff940b7962766483b28d1008c073 into 107642cce0d4407b8d1083cafe273cbe(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:02,717 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:41:02,718 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/B, priority=13, startTime=1732639262264; duration=0sec 2024-11-26T16:41:02,718 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/A, priority=13, startTime=1732639262264; duration=0sec 2024-11-26T16:41:02,718 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:02,718 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:B 2024-11-26T16:41:02,718 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:02,718 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:02,718 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:A 2024-11-26T16:41:02,719 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:02,719 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/C is initiating minor compaction (all files) 2024-11-26T16:41:02,719 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/C in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:02,720 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/f3f2c76648cd4089b3ae17a6f6ebb544, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/e9d0f863c6804ce9815d9835d8af1cfd, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/3eb8d2c296ce4c55ac8e83c1a669f54d] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=37.0 K 2024-11-26T16:41:02,722 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting f3f2c76648cd4089b3ae17a6f6ebb544, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732639259445 2024-11-26T16:41:02,722 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting e9d0f863c6804ce9815d9835d8af1cfd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1732639260150 2024-11-26T16:41:02,723 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eb8d2c296ce4c55ac8e83c1a669f54d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1732639261744 2024-11-26T16:41:02,738 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#C#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:02,739 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/cd8726db61214d218b8726b3ab7461f3 is 50, key is test_row_0/C:col10/1732639261744/Put/seqid=0 2024-11-26T16:41:02,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741936_1112 (size=13391) 2024-11-26T16:41:02,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:02,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52758 deadline: 1732639322871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:02,873 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_0' in 7c6dff940b7962766483b28d1008c073 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:41:02,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:02,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:02,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52792 deadline: 1732639322901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:02,906 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_0' in 7c6dff940b7962766483b28d1008c073 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18318 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:41:02,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:02,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52818 deadline: 1732639322921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:02,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639322922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:02,923 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in 7c6dff940b7962766483b28d1008c073 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18333 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:41:02,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:02,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639322924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:02,961 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/7bc5f9a906a744a9a53ab82da4cba063 2024-11-26T16:41:02,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/14e8080121d04098ad74e6b469c68993 is 50, key is test_row_0/B:col10/1732639261868/Put/seqid=0 2024-11-26T16:41:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T16:41:03,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741937_1113 (size=12301) 2024-11-26T16:41:03,117 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-26T16:41:03,117 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:41:03,118 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-26T16:41:03,118 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:41:03,124 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:03,124 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:03,124 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:41:03,125 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:03,125 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:03,126 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:41:03,127 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-26T16:41:03,127 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:41:03,127 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:03,127 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:03,127 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:41:03,135 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-26T16:41:03,135 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:41:03,135 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:03,135 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:03,135 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:41:03,156 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/cd8726db61214d218b8726b3ab7461f3 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/cd8726db61214d218b8726b3ab7461f3 2024-11-26T16:41:03,164 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/C of 7c6dff940b7962766483b28d1008c073 into cd8726db61214d218b8726b3ab7461f3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:03,164 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:41:03,164 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/C, priority=13, startTime=1732639262264; duration=0sec 2024-11-26T16:41:03,164 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:03,164 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:C 2024-11-26T16:41:03,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:03,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639323231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639323231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:03,447 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/14e8080121d04098ad74e6b469c68993 2024-11-26T16:41:03,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/4bb0bef88c33484fa87e94e37a1081f3 is 50, key is test_row_0/C:col10/1732639261868/Put/seqid=0 2024-11-26T16:41:03,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741938_1114 (size=12301) 2024-11-26T16:41:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T16:41:03,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:03,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52742 deadline: 1732639323741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52790 deadline: 1732639323741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:03,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:03,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:03,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:03,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:03,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:03,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:03,868 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/4bb0bef88c33484fa87e94e37a1081f3 2024-11-26T16:41:03,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/7bc5f9a906a744a9a53ab82da4cba063 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/7bc5f9a906a744a9a53ab82da4cba063 2024-11-26T16:41:03,880 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/7bc5f9a906a744a9a53ab82da4cba063, entries=150, sequenceid=521, filesize=12.0 K 2024-11-26T16:41:03,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/14e8080121d04098ad74e6b469c68993 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/14e8080121d04098ad74e6b469c68993 2024-11-26T16:41:03,888 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/14e8080121d04098ad74e6b469c68993, entries=150, sequenceid=521, filesize=12.0 K 2024-11-26T16:41:03,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/4bb0bef88c33484fa87e94e37a1081f3 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/4bb0bef88c33484fa87e94e37a1081f3 2024-11-26T16:41:03,895 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/4bb0bef88c33484fa87e94e37a1081f3, entries=150, sequenceid=521, filesize=12.0 K 2024-11-26T16:41:03,896 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~370.09 KB/378976, currentSize=53.67 KB/54960 for 7c6dff940b7962766483b28d1008c073 in 1352ms, sequenceid=521, compaction requested=false 2024-11-26T16:41:03,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:41:03,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:03,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-26T16:41:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-26T16:41:03,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-26T16:41:03,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5100 sec 2024-11-26T16:41:03,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.5180 sec 2024-11-26T16:41:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-26T16:41:04,532 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:04,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 7c6dff940b7962766483b28d1008c073 2024-11-26T16:41:04,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T16:41:04,753 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-26T16:41:04,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:41:04,753 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:41:04,753 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:04,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:04,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:41:04,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:04,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:41:04,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:04,753 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:04,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:41:04,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/1834297967864c5bb2ab111a241c2a8b is 50, key is test_row_0/A:col10/1732639262597/Put/seqid=0 2024-11-26T16:41:04,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741939_1115 (size=12301) 2024-11-26T16:41:04,763 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-26T16:41:04,763 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:41:04,763 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:04,764 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:04,764 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:41:05,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/1834297967864c5bb2ab111a241c2a8b 2024-11-26T16:41:05,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/6a8ed64287944d9b818f76e1025cb136 is 50, key is test_row_0/B:col10/1732639262597/Put/seqid=0 2024-11-26T16:41:05,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741940_1116 (size=12301) 2024-11-26T16:41:05,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/6a8ed64287944d9b818f76e1025cb136 2024-11-26T16:41:05,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/583954bcc0054332a82396c5e8bd0ac6 is 50, key is test_row_0/C:col10/1732639262597/Put/seqid=0 2024-11-26T16:41:05,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741941_1117 (size=12301) 2024-11-26T16:41:05,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/583954bcc0054332a82396c5e8bd0ac6 2024-11-26T16:41:05,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/1834297967864c5bb2ab111a241c2a8b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/1834297967864c5bb2ab111a241c2a8b 2024-11-26T16:41:05,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/1834297967864c5bb2ab111a241c2a8b, entries=150, sequenceid=536, filesize=12.0 K 2024-11-26T16:41:05,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/6a8ed64287944d9b818f76e1025cb136 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/6a8ed64287944d9b818f76e1025cb136 2024-11-26T16:41:05,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/6a8ed64287944d9b818f76e1025cb136, entries=150, sequenceid=536, filesize=12.0 K 2024-11-26T16:41:05,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/583954bcc0054332a82396c5e8bd0ac6 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/583954bcc0054332a82396c5e8bd0ac6 2024-11-26T16:41:05,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/583954bcc0054332a82396c5e8bd0ac6, entries=150, sequenceid=536, filesize=12.0 K 2024-11-26T16:41:05,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=6.71 KB/6870 for 7c6dff940b7962766483b28d1008c073 in 865ms, sequenceid=536, compaction requested=true 2024-11-26T16:41:05,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:41:05,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:41:05,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:05,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:41:05,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:05,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7c6dff940b7962766483b28d1008c073:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:41:05,619 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:05,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:05,619 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:05,620 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:05,620 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:05,620 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/A is initiating minor compaction (all files) 2024-11-26T16:41:05,620 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/B is initiating minor compaction (all files) 2024-11-26T16:41:05,620 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/A in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:05,620 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/B in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:05,620 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/107642cce0d4407b8d1083cafe273cbe, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/14e8080121d04098ad74e6b469c68993, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/6a8ed64287944d9b818f76e1025cb136] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=37.1 K 2024-11-26T16:41:05,620 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/52b6d06714bd4469a970fb5f00fa22c2, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/7bc5f9a906a744a9a53ab82da4cba063, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/1834297967864c5bb2ab111a241c2a8b] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=37.1 K 2024-11-26T16:41:05,621 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52b6d06714bd4469a970fb5f00fa22c2, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1732639261744 2024-11-26T16:41:05,621 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 107642cce0d4407b8d1083cafe273cbe, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1732639261744 2024-11-26T16:41:05,621 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bc5f9a906a744a9a53ab82da4cba063, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732639261868 2024-11-26T16:41:05,621 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 14e8080121d04098ad74e6b469c68993, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732639261868 2024-11-26T16:41:05,621 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1834297967864c5bb2ab111a241c2a8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=536, earliestPutTs=1732639262585 2024-11-26T16:41:05,622 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a8ed64287944d9b818f76e1025cb136, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=536, earliestPutTs=1732639262585 2024-11-26T16:41:05,631 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#B#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:05,632 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/13eb1f2bb26d4f1c9fc6384bbd11fa34 is 50, key is test_row_0/B:col10/1732639262597/Put/seqid=0 2024-11-26T16:41:05,634 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#A#compaction#106 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:05,635 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/ca54ed9559c24ce19f5b73b2975061b1 is 50, key is test_row_0/A:col10/1732639262597/Put/seqid=0 2024-11-26T16:41:05,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741942_1118 (size=13493) 2024-11-26T16:41:05,646 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/13eb1f2bb26d4f1c9fc6384bbd11fa34 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/13eb1f2bb26d4f1c9fc6384bbd11fa34 2024-11-26T16:41:05,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741943_1119 (size=13493) 2024-11-26T16:41:05,656 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/B of 7c6dff940b7962766483b28d1008c073 into 13eb1f2bb26d4f1c9fc6384bbd11fa34(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:05,656 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:41:05,656 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/B, priority=13, startTime=1732639265619; duration=0sec 2024-11-26T16:41:05,656 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:05,656 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:B 2024-11-26T16:41:05,656 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:05,657 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:05,658 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 7c6dff940b7962766483b28d1008c073/C is initiating minor compaction (all files) 2024-11-26T16:41:05,658 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7c6dff940b7962766483b28d1008c073/C in TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:05,658 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/cd8726db61214d218b8726b3ab7461f3, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/4bb0bef88c33484fa87e94e37a1081f3, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/583954bcc0054332a82396c5e8bd0ac6] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp, totalSize=37.1 K 2024-11-26T16:41:05,658 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting cd8726db61214d218b8726b3ab7461f3, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1732639261744 2024-11-26T16:41:05,659 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bb0bef88c33484fa87e94e37a1081f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732639261868 2024-11-26T16:41:05,659 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 583954bcc0054332a82396c5e8bd0ac6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=536, earliestPutTs=1732639262585 2024-11-26T16:41:05,669 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7c6dff940b7962766483b28d1008c073#C#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:05,670 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/b04e1b22ed9a424fbd8185c6ba20038e is 50, key is test_row_0/C:col10/1732639262597/Put/seqid=0 2024-11-26T16:41:05,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741944_1120 (size=13493) 2024-11-26T16:41:06,057 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/ca54ed9559c24ce19f5b73b2975061b1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/ca54ed9559c24ce19f5b73b2975061b1 2024-11-26T16:41:06,064 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/A of 7c6dff940b7962766483b28d1008c073 into ca54ed9559c24ce19f5b73b2975061b1(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:06,064 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:41:06,064 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/A, priority=13, startTime=1732639265619; duration=0sec 2024-11-26T16:41:06,065 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:06,065 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:A 2024-11-26T16:41:06,082 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/b04e1b22ed9a424fbd8185c6ba20038e as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/b04e1b22ed9a424fbd8185c6ba20038e 2024-11-26T16:41:06,088 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7c6dff940b7962766483b28d1008c073/C of 7c6dff940b7962766483b28d1008c073 into b04e1b22ed9a424fbd8185c6ba20038e(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:06,088 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7c6dff940b7962766483b28d1008c073: 2024-11-26T16:41:06,088 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., storeName=7c6dff940b7962766483b28d1008c073/C, priority=13, startTime=1732639265619; duration=0sec 2024-11-26T16:41:06,088 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:06,088 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7c6dff940b7962766483b28d1008c073:C 2024-11-26T16:41:06,304 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T16:41:12,923 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-26T16:41:12,924 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:41:12,924 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:12,924 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:12,924 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:41:12,954 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-26T16:41:12,954 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:41:12,954 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:12,954 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:12,954 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:41:12,994 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-26T16:41:12,994 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:41:12,994 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:12,994 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:12,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:41:12,994 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T16:41:12,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 202 2024-11-26T16:41:12,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-26T16:41:12,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 181 2024-11-26T16:41:12,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-26T16:41:12,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-26T16:41:12,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T16:41:12,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2240 2024-11-26T16:41:12,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2225 2024-11-26T16:41:12,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T16:41:12,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1047 2024-11-26T16:41:12,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3134 rows 2024-11-26T16:41:12,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1045 2024-11-26T16:41:12,995 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3131 rows 2024-11-26T16:41:12,995 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T16:41:12,995 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T16:41:12,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:12,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:12,996 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:41:13,001 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T16:41:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T16:41:13,012 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639273011"}]},"ts":"1732639273011"} 2024-11-26T16:41:13,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T16:41:13,016 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T16:41:13,017 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T16:41:13,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T16:41:13,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c6dff940b7962766483b28d1008c073, UNASSIGN}] 2024-11-26T16:41:13,027 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c6dff940b7962766483b28d1008c073, UNASSIGN 2024-11-26T16:41:13,029 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=7c6dff940b7962766483b28d1008c073, regionState=CLOSING, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:13,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c6dff940b7962766483b28d1008c073, UNASSIGN because future has completed 2024-11-26T16:41:13,033 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T16:41:13,033 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134}] 2024-11-26T16:41:13,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T16:41:13,191 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 7c6dff940b7962766483b28d1008c073 2024-11-26T16:41:13,191 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T16:41:13,192 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 7c6dff940b7962766483b28d1008c073, disabling compactions & flushes 2024-11-26T16:41:13,192 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:13,192 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:13,192 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. after waiting 0 ms 2024-11-26T16:41:13,192 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:13,193 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 7c6dff940b7962766483b28d1008c073 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-26T16:41:13,193 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=A 2024-11-26T16:41:13,193 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:13,193 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=B 2024-11-26T16:41:13,193 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:13,193 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7c6dff940b7962766483b28d1008c073, store=C 2024-11-26T16:41:13,193 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:13,198 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/d3b66c74952a460c9a989d83c0c55961 is 50, key is test_row_0/A:col10/1732639272952/Put/seqid=0 2024-11-26T16:41:13,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741945_1121 (size=9857) 2024-11-26T16:41:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T16:41:13,604 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/d3b66c74952a460c9a989d83c0c55961 2024-11-26T16:41:13,612 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/1829abb65e1e4df3b9af808322cd53e1 is 50, key is test_row_0/B:col10/1732639272952/Put/seqid=0 2024-11-26T16:41:13,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741946_1122 (size=9857) 2024-11-26T16:41:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T16:41:14,017 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/1829abb65e1e4df3b9af808322cd53e1 2024-11-26T16:41:14,026 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/1cba9d02069c45d9ad4eb3d52933d26b is 50, key is test_row_0/C:col10/1732639272952/Put/seqid=0 2024-11-26T16:41:14,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741947_1123 (size=9857) 2024-11-26T16:41:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T16:41:14,438 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/1cba9d02069c45d9ad4eb3d52933d26b 2024-11-26T16:41:14,445 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/A/d3b66c74952a460c9a989d83c0c55961 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/d3b66c74952a460c9a989d83c0c55961 2024-11-26T16:41:14,450 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/d3b66c74952a460c9a989d83c0c55961, entries=100, sequenceid=546, filesize=9.6 K 2024-11-26T16:41:14,451 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/B/1829abb65e1e4df3b9af808322cd53e1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/1829abb65e1e4df3b9af808322cd53e1 2024-11-26T16:41:14,456 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/1829abb65e1e4df3b9af808322cd53e1, entries=100, sequenceid=546, filesize=9.6 K 2024-11-26T16:41:14,457 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/.tmp/C/1cba9d02069c45d9ad4eb3d52933d26b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/1cba9d02069c45d9ad4eb3d52933d26b 2024-11-26T16:41:14,462 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/1cba9d02069c45d9ad4eb3d52933d26b, entries=100, sequenceid=546, filesize=9.6 K 2024-11-26T16:41:14,463 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 7c6dff940b7962766483b28d1008c073 in 1271ms, sequenceid=546, compaction requested=false 2024-11-26T16:41:14,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/57e15692b95d45f2b3c24dbdbeae9473, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/d76ff77803b34044acd81258000e957a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/611dc92837194c13ba0765006fa23697, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/a446dfb2db594cde8c13c0d2d12c027d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2877dec8bb8c48d686365672c33daff4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2c24f84ea6ff4e688528b8c8c735ed22, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/d3a965695ec64677ac94c6fe8adaf89b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/0407a74ddcf94ba99b7c8c5ca19b02fb, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/dd6b3892469a45aa8cccc1143bb2f773, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/f81739ddcb894addb8be6703d80499ce, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/98f8196b7d4d4651a5848c1151d324de, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/c4dc25060d1b436eaefef86cbd040be1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/4fe70b3fc2754ae89c04e620aa4cbd7a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/e40c9a7512694a53b93cdcda0112209e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/05a16c2dc03c47138adb361e67f37241, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/43fd5fe3ce7f42b48f9518148937adb6, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/da5124977fa946a0b0c5ecc1dc2923ce, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/ef35824f21f34dc4a8665bafa94667b5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/61b6f787db084d89b6cfb50174073729, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2e2ff154098248e6817977e88c17bd78, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/ce7bf6c118e548b7adf00fb9136e3fe0, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/779241c50dba4376ac9ee334c8e29c72, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/de44993be407454894db6c6acc87a5de, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/0b4c866f9bf34527ae58fd7ba21cc13a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/e1d5c816299846d2b81c2bae65fb39c0, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/67662f8fe9574632859f3695ebcd9164, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/dd89362028eb4ff7bb19eb714fb6a557, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/b91293270ce345fbb8e2300a347a447c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/27773d4cb7224737b32025ac011433a7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2da1c3203da0465f8c64d0ee328ad612, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/7ef8403d3f12435cad321028b995b89f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/52b6d06714bd4469a970fb5f00fa22c2, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/69bff1aa616a4b62ae57682a9d2fe3f5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/7bc5f9a906a744a9a53ab82da4cba063, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/1834297967864c5bb2ab111a241c2a8b] to archive 2024-11-26T16:41:14,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T16:41:14,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/57e15692b95d45f2b3c24dbdbeae9473 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/57e15692b95d45f2b3c24dbdbeae9473 2024-11-26T16:41:14,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/d76ff77803b34044acd81258000e957a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/d76ff77803b34044acd81258000e957a 2024-11-26T16:41:14,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/611dc92837194c13ba0765006fa23697 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/611dc92837194c13ba0765006fa23697 2024-11-26T16:41:14,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/a446dfb2db594cde8c13c0d2d12c027d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/a446dfb2db594cde8c13c0d2d12c027d 2024-11-26T16:41:14,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2877dec8bb8c48d686365672c33daff4 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2877dec8bb8c48d686365672c33daff4 2024-11-26T16:41:14,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2c24f84ea6ff4e688528b8c8c735ed22 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2c24f84ea6ff4e688528b8c8c735ed22 2024-11-26T16:41:14,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/d3a965695ec64677ac94c6fe8adaf89b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/d3a965695ec64677ac94c6fe8adaf89b 2024-11-26T16:41:14,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/0407a74ddcf94ba99b7c8c5ca19b02fb to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/0407a74ddcf94ba99b7c8c5ca19b02fb 2024-11-26T16:41:14,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/dd6b3892469a45aa8cccc1143bb2f773 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/dd6b3892469a45aa8cccc1143bb2f773 2024-11-26T16:41:14,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/f81739ddcb894addb8be6703d80499ce to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/f81739ddcb894addb8be6703d80499ce 2024-11-26T16:41:14,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/98f8196b7d4d4651a5848c1151d324de to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/98f8196b7d4d4651a5848c1151d324de 2024-11-26T16:41:14,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/c4dc25060d1b436eaefef86cbd040be1 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/c4dc25060d1b436eaefef86cbd040be1 2024-11-26T16:41:14,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/4fe70b3fc2754ae89c04e620aa4cbd7a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/4fe70b3fc2754ae89c04e620aa4cbd7a 2024-11-26T16:41:14,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/e40c9a7512694a53b93cdcda0112209e to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/e40c9a7512694a53b93cdcda0112209e 2024-11-26T16:41:14,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/05a16c2dc03c47138adb361e67f37241 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/05a16c2dc03c47138adb361e67f37241 2024-11-26T16:41:14,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/43fd5fe3ce7f42b48f9518148937adb6 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/43fd5fe3ce7f42b48f9518148937adb6 2024-11-26T16:41:14,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/da5124977fa946a0b0c5ecc1dc2923ce to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/da5124977fa946a0b0c5ecc1dc2923ce 2024-11-26T16:41:14,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/ef35824f21f34dc4a8665bafa94667b5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/ef35824f21f34dc4a8665bafa94667b5 2024-11-26T16:41:14,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/61b6f787db084d89b6cfb50174073729 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/61b6f787db084d89b6cfb50174073729 2024-11-26T16:41:14,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2e2ff154098248e6817977e88c17bd78 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2e2ff154098248e6817977e88c17bd78 2024-11-26T16:41:14,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/ce7bf6c118e548b7adf00fb9136e3fe0 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/ce7bf6c118e548b7adf00fb9136e3fe0 2024-11-26T16:41:14,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/779241c50dba4376ac9ee334c8e29c72 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/779241c50dba4376ac9ee334c8e29c72 2024-11-26T16:41:14,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/de44993be407454894db6c6acc87a5de to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/de44993be407454894db6c6acc87a5de 2024-11-26T16:41:14,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/0b4c866f9bf34527ae58fd7ba21cc13a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/0b4c866f9bf34527ae58fd7ba21cc13a 2024-11-26T16:41:14,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/e1d5c816299846d2b81c2bae65fb39c0 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/e1d5c816299846d2b81c2bae65fb39c0 2024-11-26T16:41:14,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/67662f8fe9574632859f3695ebcd9164 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/67662f8fe9574632859f3695ebcd9164 2024-11-26T16:41:14,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/dd89362028eb4ff7bb19eb714fb6a557 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/dd89362028eb4ff7bb19eb714fb6a557 2024-11-26T16:41:14,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/b91293270ce345fbb8e2300a347a447c to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/b91293270ce345fbb8e2300a347a447c 2024-11-26T16:41:14,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/27773d4cb7224737b32025ac011433a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/27773d4cb7224737b32025ac011433a7 2024-11-26T16:41:14,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2da1c3203da0465f8c64d0ee328ad612 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/2da1c3203da0465f8c64d0ee328ad612 2024-11-26T16:41:14,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/7ef8403d3f12435cad321028b995b89f to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/7ef8403d3f12435cad321028b995b89f 2024-11-26T16:41:14,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/52b6d06714bd4469a970fb5f00fa22c2 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/52b6d06714bd4469a970fb5f00fa22c2 2024-11-26T16:41:14,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/69bff1aa616a4b62ae57682a9d2fe3f5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/69bff1aa616a4b62ae57682a9d2fe3f5 2024-11-26T16:41:14,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/7bc5f9a906a744a9a53ab82da4cba063 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/7bc5f9a906a744a9a53ab82da4cba063 2024-11-26T16:41:14,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/1834297967864c5bb2ab111a241c2a8b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/1834297967864c5bb2ab111a241c2a8b 2024-11-26T16:41:14,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/281066a74a13447ab789ca5d9e973913, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/3ee7d070623e4f6b9245c08b4d23f881, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/8e4f5faebd834c2eb2bc3cea0ee41bc4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/7377c20524ce4bb78991200b10234ff1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/cf0acafc4567495aadd676b98fc8a63d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/001fda3ce6704a6fb82061ef2f51e433, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/e2b33f19efdb4aac81fcc6bbf81e9b2b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/5e2a95db35244527827fd0f2df4d1602, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/cfeb94040aa74e43a61fa983e903fece, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/7f0445b2c3a94661a02dbdc0b8e5c971, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/92bb23045ab64cc99acd3fccc3d58c4b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/1523339a76794901a640d8d881279578, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/41579a9c45ae4dc2a0992281755fdb0d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/e19262cbfcdd44bfbbcbfb4828e98542, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/9522f6f6bb2f4c45abb016e514972c03, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/18c81b7c41d346fe9f8b9dfb107b2d4d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/1e2af255a7f2420aac06197257670e29, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/895ca52adf134fff992fd088ae72f0e8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/e063e27cd0ef4ae781e85a3cf60bdfc0, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/0a20b5332e02444ab530ee04ee310ac7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/3928da453abe4ecda31bdf159bb1d0db, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/7503a7a06438456cb2567d02b45c35ce, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/fe9935789406457eab5932fdf9b40b4f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/fbf18f618e1f4ae891076ef2700fd9a3, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/d7f4065d76d54a629e21aed53c4be20f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/8b4a56bd7b664b6dba232f2b4c92ac77, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/3bf74dd21ed34ec785d0c7745197a470, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/65aa5fcfffd24e6995d1c02d454a91d5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/9279378fde12495fb84b892708b195d4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/4815a6fe89ef48ceb8c52b86732721c0, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/24e06f353b2c4a35a926bae23d3157f1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/107642cce0d4407b8d1083cafe273cbe, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/c019e9b6d12b4861b67b4c5e6af913b7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/14e8080121d04098ad74e6b469c68993, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/6a8ed64287944d9b818f76e1025cb136] to archive 2024-11-26T16:41:14,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T16:41:14,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/281066a74a13447ab789ca5d9e973913 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/281066a74a13447ab789ca5d9e973913 2024-11-26T16:41:14,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/3ee7d070623e4f6b9245c08b4d23f881 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/3ee7d070623e4f6b9245c08b4d23f881 2024-11-26T16:41:14,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/8e4f5faebd834c2eb2bc3cea0ee41bc4 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/8e4f5faebd834c2eb2bc3cea0ee41bc4 2024-11-26T16:41:14,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/7377c20524ce4bb78991200b10234ff1 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/7377c20524ce4bb78991200b10234ff1 2024-11-26T16:41:14,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/cf0acafc4567495aadd676b98fc8a63d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/cf0acafc4567495aadd676b98fc8a63d 2024-11-26T16:41:14,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/001fda3ce6704a6fb82061ef2f51e433 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/001fda3ce6704a6fb82061ef2f51e433 2024-11-26T16:41:14,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/e2b33f19efdb4aac81fcc6bbf81e9b2b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/e2b33f19efdb4aac81fcc6bbf81e9b2b 2024-11-26T16:41:14,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/5e2a95db35244527827fd0f2df4d1602 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/5e2a95db35244527827fd0f2df4d1602 2024-11-26T16:41:14,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/cfeb94040aa74e43a61fa983e903fece to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/cfeb94040aa74e43a61fa983e903fece 2024-11-26T16:41:14,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/7f0445b2c3a94661a02dbdc0b8e5c971 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/7f0445b2c3a94661a02dbdc0b8e5c971 2024-11-26T16:41:14,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/92bb23045ab64cc99acd3fccc3d58c4b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/92bb23045ab64cc99acd3fccc3d58c4b 2024-11-26T16:41:14,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/1523339a76794901a640d8d881279578 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/1523339a76794901a640d8d881279578 2024-11-26T16:41:14,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/41579a9c45ae4dc2a0992281755fdb0d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/41579a9c45ae4dc2a0992281755fdb0d 2024-11-26T16:41:14,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/e19262cbfcdd44bfbbcbfb4828e98542 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/e19262cbfcdd44bfbbcbfb4828e98542 2024-11-26T16:41:14,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/9522f6f6bb2f4c45abb016e514972c03 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/9522f6f6bb2f4c45abb016e514972c03 2024-11-26T16:41:14,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/18c81b7c41d346fe9f8b9dfb107b2d4d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/18c81b7c41d346fe9f8b9dfb107b2d4d 2024-11-26T16:41:14,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/1e2af255a7f2420aac06197257670e29 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/1e2af255a7f2420aac06197257670e29 2024-11-26T16:41:14,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/895ca52adf134fff992fd088ae72f0e8 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/895ca52adf134fff992fd088ae72f0e8 2024-11-26T16:41:14,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/e063e27cd0ef4ae781e85a3cf60bdfc0 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/e063e27cd0ef4ae781e85a3cf60bdfc0 2024-11-26T16:41:14,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/0a20b5332e02444ab530ee04ee310ac7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/0a20b5332e02444ab530ee04ee310ac7 2024-11-26T16:41:14,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/3928da453abe4ecda31bdf159bb1d0db to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/3928da453abe4ecda31bdf159bb1d0db 2024-11-26T16:41:14,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/7503a7a06438456cb2567d02b45c35ce to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/7503a7a06438456cb2567d02b45c35ce 2024-11-26T16:41:14,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/fe9935789406457eab5932fdf9b40b4f to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/fe9935789406457eab5932fdf9b40b4f 2024-11-26T16:41:14,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/fbf18f618e1f4ae891076ef2700fd9a3 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/fbf18f618e1f4ae891076ef2700fd9a3 2024-11-26T16:41:14,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/d7f4065d76d54a629e21aed53c4be20f to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/d7f4065d76d54a629e21aed53c4be20f 2024-11-26T16:41:14,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/8b4a56bd7b664b6dba232f2b4c92ac77 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/8b4a56bd7b664b6dba232f2b4c92ac77 2024-11-26T16:41:14,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/3bf74dd21ed34ec785d0c7745197a470 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/3bf74dd21ed34ec785d0c7745197a470 2024-11-26T16:41:14,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/65aa5fcfffd24e6995d1c02d454a91d5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/65aa5fcfffd24e6995d1c02d454a91d5 2024-11-26T16:41:14,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/9279378fde12495fb84b892708b195d4 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/9279378fde12495fb84b892708b195d4 2024-11-26T16:41:14,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/4815a6fe89ef48ceb8c52b86732721c0 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/4815a6fe89ef48ceb8c52b86732721c0 2024-11-26T16:41:14,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/24e06f353b2c4a35a926bae23d3157f1 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/24e06f353b2c4a35a926bae23d3157f1 2024-11-26T16:41:14,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/107642cce0d4407b8d1083cafe273cbe to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/107642cce0d4407b8d1083cafe273cbe 2024-11-26T16:41:14,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/c019e9b6d12b4861b67b4c5e6af913b7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/c019e9b6d12b4861b67b4c5e6af913b7 2024-11-26T16:41:14,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/14e8080121d04098ad74e6b469c68993 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/14e8080121d04098ad74e6b469c68993 2024-11-26T16:41:14,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/6a8ed64287944d9b818f76e1025cb136 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/6a8ed64287944d9b818f76e1025cb136 2024-11-26T16:41:14,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/49b91840b6ef40ea802b12272e8a2c4c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/4f3d52925f2a40f78c7261c403708cb3, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/0ec213e08b904952839187f8da3a083d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/adf4a3d77bbf4ad0a2eb0ebdc296a0e9, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/386d7a5c1765440688692c716df36723, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/e9e5b95057e14d898542688396da6408, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/a66277bb1724469b949fe1149616c7b0, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/955df0cba5074116adcae4a7ac22d5f2, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/ac6a6d22961440ea9027dc6e5d739f81, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/af859a2f1e634b2fa55e71a69ed15e4a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/377ea084cce24b609a770f9816083cc8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/2340fc53e7a84c70930207e945b8357c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/d8e4b6769a694820b1fffd961906a086, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/b44e0172af184bf8800b70fc3f3c9462, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/12ee76fc808343069ebd2dac990019ac, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/f834988a7d7b441cb01b807685a610fb, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/f0dab98f97db4263af49185d1e0500f8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/dca67aa262774aec9817597b21ddf875, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/d9112c762eb343639178483409497075, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/4bfd473c4ee44e77a1c2b729f5260334, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/a9f0a3fcf3b0458ab097002c455a0d3f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/36d97fec48194ed49e35cae445975adc, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/35fed45635e849cd9b11a49ef3eec082, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/5431feab617c4b39843323bc95199572, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/1e02d551d46c4043b474e9ce803546f5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/8029d31c8c7049ee87dc0233aeb1c371, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/9d00fb94b51a4b13bf70226de663795d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/7bd505b89e5e4e99a7f6647eeb81239c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/f3f2c76648cd4089b3ae17a6f6ebb544, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/da9d5f17761347d4a43f0799ee0f7660, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/e9d0f863c6804ce9815d9835d8af1cfd, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/cd8726db61214d218b8726b3ab7461f3, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/3eb8d2c296ce4c55ac8e83c1a669f54d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/4bb0bef88c33484fa87e94e37a1081f3, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/583954bcc0054332a82396c5e8bd0ac6] to archive 2024-11-26T16:41:14,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T16:41:14,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/49b91840b6ef40ea802b12272e8a2c4c to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/49b91840b6ef40ea802b12272e8a2c4c 2024-11-26T16:41:14,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/4f3d52925f2a40f78c7261c403708cb3 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/4f3d52925f2a40f78c7261c403708cb3 2024-11-26T16:41:14,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/0ec213e08b904952839187f8da3a083d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/0ec213e08b904952839187f8da3a083d 2024-11-26T16:41:14,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/adf4a3d77bbf4ad0a2eb0ebdc296a0e9 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/adf4a3d77bbf4ad0a2eb0ebdc296a0e9 2024-11-26T16:41:14,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/386d7a5c1765440688692c716df36723 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/386d7a5c1765440688692c716df36723 2024-11-26T16:41:14,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/e9e5b95057e14d898542688396da6408 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/e9e5b95057e14d898542688396da6408 2024-11-26T16:41:14,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/a66277bb1724469b949fe1149616c7b0 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/a66277bb1724469b949fe1149616c7b0 2024-11-26T16:41:14,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/955df0cba5074116adcae4a7ac22d5f2 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/955df0cba5074116adcae4a7ac22d5f2 2024-11-26T16:41:14,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/ac6a6d22961440ea9027dc6e5d739f81 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/ac6a6d22961440ea9027dc6e5d739f81 2024-11-26T16:41:14,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/af859a2f1e634b2fa55e71a69ed15e4a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/af859a2f1e634b2fa55e71a69ed15e4a 2024-11-26T16:41:14,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/377ea084cce24b609a770f9816083cc8 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/377ea084cce24b609a770f9816083cc8 2024-11-26T16:41:14,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/2340fc53e7a84c70930207e945b8357c to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/2340fc53e7a84c70930207e945b8357c 2024-11-26T16:41:14,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/d8e4b6769a694820b1fffd961906a086 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/d8e4b6769a694820b1fffd961906a086 2024-11-26T16:41:14,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/b44e0172af184bf8800b70fc3f3c9462 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/b44e0172af184bf8800b70fc3f3c9462 2024-11-26T16:41:14,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/12ee76fc808343069ebd2dac990019ac to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/12ee76fc808343069ebd2dac990019ac 2024-11-26T16:41:14,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/f834988a7d7b441cb01b807685a610fb to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/f834988a7d7b441cb01b807685a610fb 2024-11-26T16:41:14,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/f0dab98f97db4263af49185d1e0500f8 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/f0dab98f97db4263af49185d1e0500f8 2024-11-26T16:41:14,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/dca67aa262774aec9817597b21ddf875 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/dca67aa262774aec9817597b21ddf875 2024-11-26T16:41:14,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/d9112c762eb343639178483409497075 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/d9112c762eb343639178483409497075 2024-11-26T16:41:14,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/4bfd473c4ee44e77a1c2b729f5260334 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/4bfd473c4ee44e77a1c2b729f5260334 2024-11-26T16:41:14,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/a9f0a3fcf3b0458ab097002c455a0d3f to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/a9f0a3fcf3b0458ab097002c455a0d3f 2024-11-26T16:41:14,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/36d97fec48194ed49e35cae445975adc to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/36d97fec48194ed49e35cae445975adc 2024-11-26T16:41:14,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/35fed45635e849cd9b11a49ef3eec082 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/35fed45635e849cd9b11a49ef3eec082 2024-11-26T16:41:14,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/5431feab617c4b39843323bc95199572 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/5431feab617c4b39843323bc95199572 2024-11-26T16:41:14,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/1e02d551d46c4043b474e9ce803546f5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/1e02d551d46c4043b474e9ce803546f5 2024-11-26T16:41:14,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/8029d31c8c7049ee87dc0233aeb1c371 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/8029d31c8c7049ee87dc0233aeb1c371 2024-11-26T16:41:14,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/9d00fb94b51a4b13bf70226de663795d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/9d00fb94b51a4b13bf70226de663795d 2024-11-26T16:41:14,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/7bd505b89e5e4e99a7f6647eeb81239c to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/7bd505b89e5e4e99a7f6647eeb81239c 2024-11-26T16:41:14,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/f3f2c76648cd4089b3ae17a6f6ebb544 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/f3f2c76648cd4089b3ae17a6f6ebb544 2024-11-26T16:41:14,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/da9d5f17761347d4a43f0799ee0f7660 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/da9d5f17761347d4a43f0799ee0f7660 2024-11-26T16:41:14,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/e9d0f863c6804ce9815d9835d8af1cfd to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/e9d0f863c6804ce9815d9835d8af1cfd 2024-11-26T16:41:14,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/cd8726db61214d218b8726b3ab7461f3 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/cd8726db61214d218b8726b3ab7461f3 2024-11-26T16:41:14,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/3eb8d2c296ce4c55ac8e83c1a669f54d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/3eb8d2c296ce4c55ac8e83c1a669f54d 2024-11-26T16:41:14,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/4bb0bef88c33484fa87e94e37a1081f3 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/4bb0bef88c33484fa87e94e37a1081f3 2024-11-26T16:41:14,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/583954bcc0054332a82396c5e8bd0ac6 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/583954bcc0054332a82396c5e8bd0ac6 2024-11-26T16:41:14,662 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/recovered.edits/549.seqid, newMaxSeqId=549, maxSeqId=1 2024-11-26T16:41:14,663 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. 2024-11-26T16:41:14,664 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 7c6dff940b7962766483b28d1008c073: Waiting for close lock at 1732639273192Running coprocessor pre-close hooks at 1732639273192Disabling compacts and flushes for region at 1732639273192Disabling writes for close at 1732639273192Obtaining lock to block concurrent updates at 1732639273193 (+1 ms)Preparing flush snapshotting stores in 7c6dff940b7962766483b28d1008c073 at 1732639273193Finished memstore snapshotting TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732639273193Flushing stores of TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073. at 1732639273194 (+1 ms)Flushing 7c6dff940b7962766483b28d1008c073/A: creating writer at 1732639273194Flushing 7c6dff940b7962766483b28d1008c073/A: appending metadata at 1732639273197 (+3 ms)Flushing 7c6dff940b7962766483b28d1008c073/A: closing flushed file at 1732639273197Flushing 7c6dff940b7962766483b28d1008c073/B: creating writer at 1732639273609 (+412 ms)Flushing 7c6dff940b7962766483b28d1008c073/B: appending metadata at 1732639273612 (+3 ms)Flushing 7c6dff940b7962766483b28d1008c073/B: closing flushed file at 1732639273612Flushing 7c6dff940b7962766483b28d1008c073/C: creating writer at 1732639274022 (+410 ms)Flushing 7c6dff940b7962766483b28d1008c073/C: appending metadata at 1732639274026 (+4 ms)Flushing 7c6dff940b7962766483b28d1008c073/C: closing flushed file at 1732639274026Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36ff3474: reopening flushed file at 1732639274443 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65b2d58a: reopening flushed file at 1732639274450 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57a008c: reopening flushed file at 1732639274456 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 7c6dff940b7962766483b28d1008c073 in 1271ms, sequenceid=546, compaction requested=false at 1732639274463 (+7 ms)Writing region close event to WAL at 1732639274652 (+189 ms)Running coprocessor post-close hooks at 1732639274663 (+11 ms)Closed at 1732639274663 2024-11-26T16:41:14,668 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 7c6dff940b7962766483b28d1008c073 2024-11-26T16:41:14,668 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=7c6dff940b7962766483b28d1008c073, regionState=CLOSED 2024-11-26T16:41:14,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 because future has completed 2024-11-26T16:41:14,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-26T16:41:14,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 7c6dff940b7962766483b28d1008c073, server=2dedfe7466e8,38109,1732639238134 in 1.6440 sec 2024-11-26T16:41:14,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-26T16:41:14,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7c6dff940b7962766483b28d1008c073, UNASSIGN in 1.6540 sec 2024-11-26T16:41:14,689 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-26T16:41:14,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6660 sec 2024-11-26T16:41:14,693 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639274693"}]},"ts":"1732639274693"} 2024-11-26T16:41:14,695 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T16:41:14,695 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T16:41:14,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6930 sec 2024-11-26T16:41:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-26T16:41:15,162 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:15,165 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T16:41:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:41:15,171 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:41:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-26T16:41:15,173 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:41:15,179 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073 2024-11-26T16:41:15,184 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A, FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B, FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C, FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/recovered.edits] 2024-11-26T16:41:15,188 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/ca54ed9559c24ce19f5b73b2975061b1 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/ca54ed9559c24ce19f5b73b2975061b1 2024-11-26T16:41:15,190 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/d3b66c74952a460c9a989d83c0c55961 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/A/d3b66c74952a460c9a989d83c0c55961 2024-11-26T16:41:15,193 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/13eb1f2bb26d4f1c9fc6384bbd11fa34 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/13eb1f2bb26d4f1c9fc6384bbd11fa34 2024-11-26T16:41:15,195 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/1829abb65e1e4df3b9af808322cd53e1 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/B/1829abb65e1e4df3b9af808322cd53e1 2024-11-26T16:41:15,199 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/1cba9d02069c45d9ad4eb3d52933d26b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/1cba9d02069c45d9ad4eb3d52933d26b 2024-11-26T16:41:15,200 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/b04e1b22ed9a424fbd8185c6ba20038e to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/C/b04e1b22ed9a424fbd8185c6ba20038e 2024-11-26T16:41:15,204 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/recovered.edits/549.seqid to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073/recovered.edits/549.seqid 2024-11-26T16:41:15,205 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/7c6dff940b7962766483b28d1008c073 2024-11-26T16:41:15,205 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T16:41:15,216 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:41:15,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-26T16:41:15,227 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T16:41:15,259 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T16:41:15,261 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:41:15,261 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T16:41:15,262 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732639275261"}]},"ts":"9223372036854775807"} 2024-11-26T16:41:15,266 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T16:41:15,266 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7c6dff940b7962766483b28d1008c073, NAME => 'TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T16:41:15,267 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T16:41:15,267 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732639275267"}]},"ts":"9223372036854775807"} 2024-11-26T16:41:15,271 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T16:41:15,272 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:41:15,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 106 msec 2024-11-26T16:41:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-26T16:41:15,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T16:41:15,282 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:15,294 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;2dedfe7466e8:38109-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2059946220_22 at /127.0.0.1:34820 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=390 (was 275) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6537 (was 6520) - AvailableMemoryMB LEAK? - 2024-11-26T16:41:15,305 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=390, ProcessCount=11, AvailableMemoryMB=6537 2024-11-26T16:41:15,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T16:41:15,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T16:41:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T16:41:15,310 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T16:41:15,310 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:15,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-26T16:41:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-26T16:41:15,311 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T16:41:15,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741948_1124 (size=1061) 2024-11-26T16:41:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-26T16:41:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-26T16:41:15,726 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711 2024-11-26T16:41:15,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741949_1125 (size=53) 2024-11-26T16:41:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-26T16:41:16,139 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T16:41:16,139 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 974fac4d1c1c1475caaca19099f1def5, disabling compactions & flushes 2024-11-26T16:41:16,140 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:16,140 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:16,140 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. after waiting 0 ms 2024-11-26T16:41:16,140 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:16,140 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:16,140 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 974fac4d1c1c1475caaca19099f1def5: Waiting for close lock at 1732639276139Disabling compacts and flushes for region at 1732639276139Disabling writes for close at 1732639276140 (+1 ms)Writing region close event to WAL at 1732639276140Closed at 1732639276140 2024-11-26T16:41:16,141 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T16:41:16,142 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732639276142"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732639276142"}]},"ts":"1732639276142"} 2024-11-26T16:41:16,144 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T16:41:16,146 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T16:41:16,146 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639276146"}]},"ts":"1732639276146"} 2024-11-26T16:41:16,147 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T16:41:16,148 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=974fac4d1c1c1475caaca19099f1def5, ASSIGN}] 2024-11-26T16:41:16,149 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=974fac4d1c1c1475caaca19099f1def5, ASSIGN 2024-11-26T16:41:16,150 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=974fac4d1c1c1475caaca19099f1def5, ASSIGN; state=OFFLINE, location=2dedfe7466e8,38109,1732639238134; forceNewPlan=false, retain=false 2024-11-26T16:41:16,301 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=974fac4d1c1c1475caaca19099f1def5, regionState=OPENING, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:16,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=974fac4d1c1c1475caaca19099f1def5, ASSIGN because future has completed 2024-11-26T16:41:16,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134}] 2024-11-26T16:41:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-26T16:41:16,460 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:16,460 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} 2024-11-26T16:41:16,461 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:16,461 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T16:41:16,461 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:16,461 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:16,463 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:16,465 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:41:16,466 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 974fac4d1c1c1475caaca19099f1def5 columnFamilyName A 2024-11-26T16:41:16,466 DEBUG [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:16,466 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] regionserver.HStore(327): Store=974fac4d1c1c1475caaca19099f1def5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:41:16,467 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:16,468 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:41:16,468 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 974fac4d1c1c1475caaca19099f1def5 columnFamilyName B 2024-11-26T16:41:16,468 DEBUG [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:16,469 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] regionserver.HStore(327): Store=974fac4d1c1c1475caaca19099f1def5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:41:16,469 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:16,470 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:41:16,471 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 974fac4d1c1c1475caaca19099f1def5 columnFamilyName C 2024-11-26T16:41:16,471 DEBUG [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:16,471 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] regionserver.HStore(327): Store=974fac4d1c1c1475caaca19099f1def5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:41:16,471 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:16,471 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:16,472 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:16,472 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:16,474 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:16,474 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:16,474 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T16:41:16,476 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:16,482 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T16:41:16,482 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 974fac4d1c1c1475caaca19099f1def5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59408970, jitterRate=-0.1147373616695404}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T16:41:16,483 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:16,483 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 974fac4d1c1c1475caaca19099f1def5: Running coprocessor pre-open hook at 1732639276461Writing region info on filesystem at 1732639276461Initializing all the Stores at 1732639276463 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639276463Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639276463Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639276463Cleaning up temporary data from old regions at 1732639276474 (+11 ms)Running coprocessor post-open hooks at 1732639276483 (+9 ms)Region opened successfully at 1732639276483 2024-11-26T16:41:16,484 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., pid=32, masterSystemTime=1732639276455 2024-11-26T16:41:16,486 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:16,487 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:16,487 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=974fac4d1c1c1475caaca19099f1def5, regionState=OPEN, openSeqNum=2, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:16,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 because future has completed 2024-11-26T16:41:16,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-26T16:41:16,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 in 186 msec 2024-11-26T16:41:16,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-26T16:41:16,494 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=974fac4d1c1c1475caaca19099f1def5, ASSIGN in 343 msec 2024-11-26T16:41:16,494 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T16:41:16,494 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639276494"}]},"ts":"1732639276494"} 2024-11-26T16:41:16,496 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T16:41:16,497 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T16:41:16,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1900 sec 2024-11-26T16:41:17,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-26T16:41:17,462 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:17,463 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e30321, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:17,463 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:17,463 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:17,465 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:17,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:17,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:17,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52b434e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:17,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:17,468 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f8cfaec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:17,471 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:17,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:17,477 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T16:41:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 2dedfe7466e8,35233,1732639237322 2024-11-26T16:41:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5eaae187 2024-11-26T16:41:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T16:41:17,480 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T16:41:17,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T16:41:17,492 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T16:41:17,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T16:41:17,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741950_1126 (size=1097) 2024-11-26T16:41:17,938 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-26T16:41:17,939 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-26T16:41:17,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T16:41:17,949 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=974fac4d1c1c1475caaca19099f1def5, REOPEN/MOVE}] 2024-11-26T16:41:17,950 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=974fac4d1c1c1475caaca19099f1def5, REOPEN/MOVE 2024-11-26T16:41:17,951 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=974fac4d1c1c1475caaca19099f1def5, regionState=CLOSING, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:17,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=974fac4d1c1c1475caaca19099f1def5, REOPEN/MOVE because future has completed 2024-11-26T16:41:17,953 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T16:41:17,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134}] 2024-11-26T16:41:18,106 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:18,106 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T16:41:18,107 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 974fac4d1c1c1475caaca19099f1def5, disabling compactions & flushes 2024-11-26T16:41:18,107 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:18,107 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:18,107 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. after waiting 0 ms 2024-11-26T16:41:18,107 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:18,113 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T16:41:18,114 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:18,114 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 974fac4d1c1c1475caaca19099f1def5: Waiting for close lock at 1732639278106Running coprocessor pre-close hooks at 1732639278106Disabling compacts and flushes for region at 1732639278107 (+1 ms)Disabling writes for close at 1732639278107Writing region close event to WAL at 1732639278107Running coprocessor post-close hooks at 1732639278114 (+7 ms)Closed at 1732639278114 2024-11-26T16:41:18,115 WARN [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 974fac4d1c1c1475caaca19099f1def5 to self. 2024-11-26T16:41:18,117 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:18,118 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=974fac4d1c1c1475caaca19099f1def5, regionState=CLOSED 2024-11-26T16:41:18,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 because future has completed 2024-11-26T16:41:18,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-26T16:41:18,125 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 in 169 msec 2024-11-26T16:41:18,126 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=974fac4d1c1c1475caaca19099f1def5, REOPEN/MOVE; state=CLOSED, location=2dedfe7466e8,38109,1732639238134; forceNewPlan=false, retain=true 2024-11-26T16:41:18,276 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=974fac4d1c1c1475caaca19099f1def5, regionState=OPENING, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:18,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=974fac4d1c1c1475caaca19099f1def5, REOPEN/MOVE because future has completed 2024-11-26T16:41:18,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134}] 2024-11-26T16:41:18,435 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:18,435 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} 2024-11-26T16:41:18,436 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:18,436 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T16:41:18,436 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:18,436 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:18,440 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:18,441 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:41:18,446 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 974fac4d1c1c1475caaca19099f1def5 columnFamilyName A 2024-11-26T16:41:18,447 DEBUG [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:18,448 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] regionserver.HStore(327): Store=974fac4d1c1c1475caaca19099f1def5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:41:18,448 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:18,449 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:41:18,449 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 974fac4d1c1c1475caaca19099f1def5 columnFamilyName B 2024-11-26T16:41:18,450 DEBUG [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:18,450 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] regionserver.HStore(327): Store=974fac4d1c1c1475caaca19099f1def5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:41:18,450 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:18,451 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:41:18,451 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 974fac4d1c1c1475caaca19099f1def5 columnFamilyName C 2024-11-26T16:41:18,451 DEBUG [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:18,451 INFO [StoreOpener-974fac4d1c1c1475caaca19099f1def5-1 {}] regionserver.HStore(327): Store=974fac4d1c1c1475caaca19099f1def5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:41:18,452 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:18,452 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:18,452 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:18,453 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:18,454 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:18,454 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:18,455 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T16:41:18,456 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:18,457 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 974fac4d1c1c1475caaca19099f1def5; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72839906, jitterRate=0.08539918065071106}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T16:41:18,457 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:18,458 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 974fac4d1c1c1475caaca19099f1def5: Running coprocessor pre-open hook at 1732639278436Writing region info on filesystem at 1732639278436Initializing all the Stores at 1732639278437 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639278437Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639278437Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639278437Cleaning up temporary data from old regions at 1732639278454 (+17 ms)Running coprocessor post-open hooks at 1732639278457 (+3 ms)Region opened successfully at 1732639278458 (+1 ms) 2024-11-26T16:41:18,459 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., pid=37, masterSystemTime=1732639278431 2024-11-26T16:41:18,461 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:18,461 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:18,462 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=974fac4d1c1c1475caaca19099f1def5, regionState=OPEN, openSeqNum=5, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:18,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 because future has completed 2024-11-26T16:41:18,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-26T16:41:18,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 in 185 msec 2024-11-26T16:41:18,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-26T16:41:18,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=974fac4d1c1c1475caaca19099f1def5, REOPEN/MOVE in 517 msec 2024-11-26T16:41:18,471 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-26T16:41:18,471 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 527 msec 2024-11-26T16:41:18,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 978 msec 2024-11-26T16:41:18,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-26T16:41:18,476 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:18,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f9c3dcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,477 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:18,477 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:18,478 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:18,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:18,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:18,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1882ab66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:18,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:18,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:18,480 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:18,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3146dbb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,481 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dde8260, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,481 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:18,482 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:18,483 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:18,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:18,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:18,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a87beba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:18,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:18,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:18,484 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3748c2ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab5c81d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,486 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:18,486 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:18,486 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aa5d897, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:18,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:18,488 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a2d2ce5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ea1a430, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,489 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:18,489 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:18,490 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0ee62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:18,491 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f85f253, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fbff363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,492 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:18,492 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:18,493 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2974d94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:18,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:18,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:18,494 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:18,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bb99788, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292bddff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,496 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:18,496 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:18,496 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@243b8487, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:18,498 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:18,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39fbf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,499 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21a14e5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,499 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:18,499 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:18,499 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:18,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:18,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:18,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b642b44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:18,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:18,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:18,501 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:18,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424ecd44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f17c234, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,502 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:18,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:18,502 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:18,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef62b6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:18,504 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:18,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70104d17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197f7df3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,505 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:18,505 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:18,505 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:18,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:18,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:18,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c4e3883, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:18,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:18,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:18,507 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:18,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ba1929, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:18,508 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:41:18,508 DEBUG [Thread-654 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:18,508 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:18,508 DEBUG [Thread-652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:18,509 DEBUG [Thread-643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:18,510 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:18,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:18,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-26T16:41:18,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:18,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:18,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:18,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:18,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:18,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:18,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:18,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:18,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:18,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T16:41:18,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:18,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:18,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:18,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:18,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:18,514 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:18,515 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:18,515 DEBUG [Thread-641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:18,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:18,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:18,515 DEBUG [Thread-645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:18,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:18,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:18,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:18,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:18,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:18,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:18,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:41:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:41:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:41:18,520 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:18,520 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:18,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:18,521 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:41:18,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:41:18,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:41:18,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:41:18,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:41:18,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:41:18,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:41:18,526 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:41:18,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:41:18,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 974fac4d1c1c1475caaca19099f1def5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T16:41:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:18,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=A 2024-11-26T16:41:18,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:18,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=B 2024-11-26T16:41:18,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:18,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=C 2024-11-26T16:41:18,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:18,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112616880becc0024f449e05d7329ed8bc2c_974fac4d1c1c1475caaca19099f1def5 is 50, key is test_row_0/A:col10/1732639278536/Put/seqid=0 2024-11-26T16:41:18,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741951_1127 (size=14594) 2024-11-26T16:41:18,605 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:18,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732639278585, max timestamp=1732639278603Num uniques 859; 2024-11-26T16:41:18,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T16:41:18,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732639278585, max timestamp=1732639278603Num uniques 859; 2024-11-26T16:41:18,613 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112616880becc0024f449e05d7329ed8bc2c_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112616880becc0024f449e05d7329ed8bc2c_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:18,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T16:41:18,615 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/abf2528ef4824b65b1bda57dd128bfae, store: [table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:18,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732639278585, max timestamp=1732639278603Num uniques 859; 2024-11-26T16:41:18,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-26T16:41:18,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T16:41:18,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/abf2528ef4824b65b1bda57dd128bfae is 175, key is test_row_0/A:col10/1732639278536/Put/seqid=0 2024-11-26T16:41:18,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639338630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:18,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741952_1128 (size=39549) 2024-11-26T16:41:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639338631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:18,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639338633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:18,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639338635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639338637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:18,652 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/abf2528ef4824b65b1bda57dd128bfae 2024-11-26T16:41:18,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T16:41:18,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:18,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:18,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:18,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:18,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:18,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/d534e75cd65b4a789acdb3522adff852 is 50, key is test_row_0/B:col10/1732639278536/Put/seqid=0 2024-11-26T16:41:18,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741953_1129 (size=12001) 2024-11-26T16:41:18,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/d534e75cd65b4a789acdb3522adff852 2024-11-26T16:41:18,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/c8358d829fab49c4bdf4487ea64943f9 is 50, key is test_row_0/C:col10/1732639278536/Put/seqid=0 2024-11-26T16:41:18,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639338759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:18,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639338759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:18,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:18,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:18,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639338759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:18,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741954_1130 (size=12001) 2024-11-26T16:41:18,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:18,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639338759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639338759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:18,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/c8358d829fab49c4bdf4487ea64943f9 2024-11-26T16:41:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:18,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/abf2528ef4824b65b1bda57dd128bfae as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/abf2528ef4824b65b1bda57dd128bfae 2024-11-26T16:41:18,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/abf2528ef4824b65b1bda57dd128bfae, entries=200, sequenceid=15, filesize=38.6 K 2024-11-26T16:41:18,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/d534e75cd65b4a789acdb3522adff852 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/d534e75cd65b4a789acdb3522adff852 2024-11-26T16:41:18,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/d534e75cd65b4a789acdb3522adff852, entries=150, sequenceid=15, filesize=11.7 K 2024-11-26T16:41:18,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/c8358d829fab49c4bdf4487ea64943f9 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/c8358d829fab49c4bdf4487ea64943f9 2024-11-26T16:41:18,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/c8358d829fab49c4bdf4487ea64943f9, entries=150, sequenceid=15, filesize=11.7 K 2024-11-26T16:41:18,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 974fac4d1c1c1475caaca19099f1def5 in 265ms, sequenceid=15, compaction requested=false 2024-11-26T16:41:18,802 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T16:41:18,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T16:41:18,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-26T16:41:18,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:18,841 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 974fac4d1c1c1475caaca19099f1def5 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-26T16:41:18,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=A 2024-11-26T16:41:18,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:41:18,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=B 2024-11-26T16:41:18,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:41:18,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=C 2024-11-26T16:41:18,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:41:18,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262cf5c0c448ce4e7ab65e3a66d08b16e4_974fac4d1c1c1475caaca19099f1def5 is 50, key is test_row_0/A:col10/1732639278634/Put/seqid=0 2024-11-26T16:41:18,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741955_1131 (size=12154) 2024-11-26T16:41:18,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:18,890 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262cf5c0c448ce4e7ab65e3a66d08b16e4_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262cf5c0c448ce4e7ab65e3a66d08b16e4_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:18,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/10afa2d184b846d38779851cdabc4ecc, store: [table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:18,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/10afa2d184b846d38779851cdabc4ecc is 175, key is test_row_0/A:col10/1732639278634/Put/seqid=0 2024-11-26T16:41:18,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741956_1132 (size=30955) 2024-11-26T16:41:18,904 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/10afa2d184b846d38779851cdabc4ecc 2024-11-26T16:41:18,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/96974dc5acbc43a789fe9ebd7f38a58a is 50, key is test_row_0/B:col10/1732639278634/Put/seqid=0 2024-11-26T16:41:18,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741957_1133 (size=12001) 2024-11-26T16:41:18,936 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/96974dc5acbc43a789fe9ebd7f38a58a 2024-11-26T16:41:18,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/291d03cc66c348a18995f4d83f266aa8 is 50, key is test_row_0/C:col10/1732639278634/Put/seqid=0 2024-11-26T16:41:18,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741958_1134 (size=12001) 2024-11-26T16:41:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:18,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:18,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639338991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639338996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639338997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639338997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639338999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639339102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639339112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639339112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639339112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639339112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T16:41:19,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639339312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639339321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639339321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639339322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639339322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,366 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/291d03cc66c348a18995f4d83f266aa8 2024-11-26T16:41:19,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/10afa2d184b846d38779851cdabc4ecc as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/10afa2d184b846d38779851cdabc4ecc 2024-11-26T16:41:19,385 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/10afa2d184b846d38779851cdabc4ecc, entries=150, sequenceid=49, filesize=30.2 K 2024-11-26T16:41:19,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/96974dc5acbc43a789fe9ebd7f38a58a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/96974dc5acbc43a789fe9ebd7f38a58a 2024-11-26T16:41:19,392 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/96974dc5acbc43a789fe9ebd7f38a58a, entries=150, sequenceid=49, filesize=11.7 K 2024-11-26T16:41:19,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/291d03cc66c348a18995f4d83f266aa8 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/291d03cc66c348a18995f4d83f266aa8 2024-11-26T16:41:19,401 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/291d03cc66c348a18995f4d83f266aa8, entries=150, sequenceid=49, filesize=11.7 K 2024-11-26T16:41:19,403 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 974fac4d1c1c1475caaca19099f1def5 in 562ms, sequenceid=49, compaction requested=false 2024-11-26T16:41:19,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:19,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:19,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-26T16:41:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-26T16:41:19,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-26T16:41:19,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 878 msec 2024-11-26T16:41:19,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 901 msec 2024-11-26T16:41:19,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:19,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 974fac4d1c1c1475caaca19099f1def5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T16:41:19,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=A 2024-11-26T16:41:19,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:19,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=B 2024-11-26T16:41:19,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:19,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=C 2024-11-26T16:41:19,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:19,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268f9257e48c4a4ef2a1a68913128901bc_974fac4d1c1c1475caaca19099f1def5 is 50, key is test_row_0/A:col10/1732639279632/Put/seqid=0 2024-11-26T16:41:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-26T16:41:19,652 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:19,652 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:41:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-26T16:41:19,654 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:41:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T16:41:19,655 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:41:19,655 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:41:19,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639339673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639339674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639339676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639339678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639339680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741959_1135 (size=14594) 2024-11-26T16:41:19,703 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:19,708 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411268f9257e48c4a4ef2a1a68913128901bc_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411268f9257e48c4a4ef2a1a68913128901bc_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:19,710 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/3c8762778f06462d8579c67b07506f5b, store: [table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:19,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/3c8762778f06462d8579c67b07506f5b is 175, key is test_row_0/A:col10/1732639279632/Put/seqid=0 2024-11-26T16:41:19,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741960_1136 (size=39549) 2024-11-26T16:41:19,728 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/3c8762778f06462d8579c67b07506f5b 2024-11-26T16:41:19,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/292dfd79d68e4476a5b42d10df04e25d is 50, key is test_row_0/B:col10/1732639279632/Put/seqid=0 2024-11-26T16:41:19,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741961_1137 (size=12001) 2024-11-26T16:41:19,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/292dfd79d68e4476a5b42d10df04e25d 2024-11-26T16:41:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T16:41:19,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/8a55fb8560754e5eb665fb6e054b9942 is 50, key is test_row_0/C:col10/1732639279632/Put/seqid=0 2024-11-26T16:41:19,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741962_1138 (size=12001) 2024-11-26T16:41:19,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/8a55fb8560754e5eb665fb6e054b9942 2024-11-26T16:41:19,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/3c8762778f06462d8579c67b07506f5b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/3c8762778f06462d8579c67b07506f5b 2024-11-26T16:41:19,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639339791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639339792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639339792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/3c8762778f06462d8579c67b07506f5b, entries=200, sequenceid=61, filesize=38.6 K 2024-11-26T16:41:19,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/292dfd79d68e4476a5b42d10df04e25d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/292dfd79d68e4476a5b42d10df04e25d 2024-11-26T16:41:19,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639339801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639339801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:19,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/292dfd79d68e4476a5b42d10df04e25d, entries=150, sequenceid=61, filesize=11.7 K 2024-11-26T16:41:19,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-26T16:41:19,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:19,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:19,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:19,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/8a55fb8560754e5eb665fb6e054b9942 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/8a55fb8560754e5eb665fb6e054b9942 2024-11-26T16:41:19,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:19,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:19,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/8a55fb8560754e5eb665fb6e054b9942, entries=150, sequenceid=61, filesize=11.7 K 2024-11-26T16:41:19,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 974fac4d1c1c1475caaca19099f1def5 in 190ms, sequenceid=61, compaction requested=true 2024-11-26T16:41:19,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:19,825 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:19,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:41:19,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:19,826 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:19,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:41:19,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:19,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:41:19,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:19,827 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:19,827 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/A is initiating minor compaction (all files) 2024-11-26T16:41:19,827 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/A in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:19,827 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/abf2528ef4824b65b1bda57dd128bfae, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/10afa2d184b846d38779851cdabc4ecc, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/3c8762778f06462d8579c67b07506f5b] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=107.5 K 2024-11-26T16:41:19,827 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:19,828 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. files: [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/abf2528ef4824b65b1bda57dd128bfae, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/10afa2d184b846d38779851cdabc4ecc, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/3c8762778f06462d8579c67b07506f5b] 2024-11-26T16:41:19,829 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:19,829 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting abf2528ef4824b65b1bda57dd128bfae, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732639278524 2024-11-26T16:41:19,830 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/B is initiating minor compaction (all files) 2024-11-26T16:41:19,830 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/B in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:19,830 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/d534e75cd65b4a789acdb3522adff852, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/96974dc5acbc43a789fe9ebd7f38a58a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/292dfd79d68e4476a5b42d10df04e25d] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=35.2 K 2024-11-26T16:41:19,834 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10afa2d184b846d38779851cdabc4ecc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732639278617 2024-11-26T16:41:19,834 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting d534e75cd65b4a789acdb3522adff852, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732639278524 2024-11-26T16:41:19,835 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 96974dc5acbc43a789fe9ebd7f38a58a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732639278617 2024-11-26T16:41:19,835 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c8762778f06462d8579c67b07506f5b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732639278990 2024-11-26T16:41:19,835 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 292dfd79d68e4476a5b42d10df04e25d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732639278990 2024-11-26T16:41:19,855 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#B#compaction#120 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:19,855 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/a0709759d30f4cc381537334e699b8d6 is 50, key is test_row_0/B:col10/1732639279632/Put/seqid=0 2024-11-26T16:41:19,861 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:19,865 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126829d449fe1564b45a6048b2fd09d13d5_974fac4d1c1c1475caaca19099f1def5 store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:19,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741963_1139 (size=12104) 2024-11-26T16:41:19,874 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126829d449fe1564b45a6048b2fd09d13d5_974fac4d1c1c1475caaca19099f1def5, store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:19,874 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126829d449fe1564b45a6048b2fd09d13d5_974fac4d1c1c1475caaca19099f1def5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:19,880 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/a0709759d30f4cc381537334e699b8d6 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/a0709759d30f4cc381537334e699b8d6 2024-11-26T16:41:19,891 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/B of 974fac4d1c1c1475caaca19099f1def5 into a0709759d30f4cc381537334e699b8d6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:19,891 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:19,891 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/B, priority=13, startTime=1732639279825; duration=0sec 2024-11-26T16:41:19,891 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:19,891 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:B 2024-11-26T16:41:19,891 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:19,893 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:19,893 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/C is initiating minor compaction (all files) 2024-11-26T16:41:19,893 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/C in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:19,893 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/c8358d829fab49c4bdf4487ea64943f9, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/291d03cc66c348a18995f4d83f266aa8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/8a55fb8560754e5eb665fb6e054b9942] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=35.2 K 2024-11-26T16:41:19,894 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting c8358d829fab49c4bdf4487ea64943f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732639278524 2024-11-26T16:41:19,894 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 291d03cc66c348a18995f4d83f266aa8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732639278617 2024-11-26T16:41:19,894 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a55fb8560754e5eb665fb6e054b9942, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732639278990 2024-11-26T16:41:19,909 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#C#compaction#122 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:19,909 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/944f5ffd08ff4ebcaefd1b80f3e81273 is 50, key is test_row_0/C:col10/1732639279632/Put/seqid=0 2024-11-26T16:41:19,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741964_1140 (size=4469) 2024-11-26T16:41:19,915 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#A#compaction#121 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:19,916 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/cd61b7f25ed44987973d4aca55b20b9b is 175, key is test_row_0/A:col10/1732639279632/Put/seqid=0 2024-11-26T16:41:19,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741966_1142 (size=31058) 2024-11-26T16:41:19,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741965_1141 (size=12104) 2024-11-26T16:41:19,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-26T16:41:19,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:19,962 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 974fac4d1c1c1475caaca19099f1def5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T16:41:19,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=A 2024-11-26T16:41:19,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:19,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=B 2024-11-26T16:41:19,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:19,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=C 2024-11-26T16:41:19,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T16:41:19,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112675a2e82da4134e008a5289937874e6c6_974fac4d1c1c1475caaca19099f1def5 is 50, key is test_row_0/A:col10/1732639279678/Put/seqid=0 2024-11-26T16:41:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:20,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:20,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741967_1143 (size=12154) 2024-11-26T16:41:20,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:20,026 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112675a2e82da4134e008a5289937874e6c6_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112675a2e82da4134e008a5289937874e6c6_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:20,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/cc30350b91c74d1dbf1b32296c42c05f, store: [table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:20,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/cc30350b91c74d1dbf1b32296c42c05f is 175, key is test_row_0/A:col10/1732639279678/Put/seqid=0 2024-11-26T16:41:20,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639340023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639340023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639340028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639340035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741968_1144 (size=30955) 2024-11-26T16:41:20,047 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/cc30350b91c74d1dbf1b32296c42c05f 2024-11-26T16:41:20,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639340041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/adf93014f9034c668b4095b8636e2060 is 50, key is test_row_0/B:col10/1732639279678/Put/seqid=0 2024-11-26T16:41:20,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741969_1145 (size=12001) 2024-11-26T16:41:20,093 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/adf93014f9034c668b4095b8636e2060 2024-11-26T16:41:20,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/80b5e407c2ba4006844ce09dcb57284b is 50, key is test_row_0/C:col10/1732639279678/Put/seqid=0 2024-11-26T16:41:20,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639340142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639340142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639340142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741970_1146 (size=12001) 2024-11-26T16:41:20,154 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/80b5e407c2ba4006844ce09dcb57284b 2024-11-26T16:41:20,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639340151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639340161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/cc30350b91c74d1dbf1b32296c42c05f as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/cc30350b91c74d1dbf1b32296c42c05f 2024-11-26T16:41:20,194 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/cc30350b91c74d1dbf1b32296c42c05f, entries=150, sequenceid=82, filesize=30.2 K 2024-11-26T16:41:20,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/adf93014f9034c668b4095b8636e2060 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/adf93014f9034c668b4095b8636e2060 2024-11-26T16:41:20,202 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/adf93014f9034c668b4095b8636e2060, entries=150, sequenceid=82, filesize=11.7 K 2024-11-26T16:41:20,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/80b5e407c2ba4006844ce09dcb57284b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/80b5e407c2ba4006844ce09dcb57284b 2024-11-26T16:41:20,212 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T16:41:20,214 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/80b5e407c2ba4006844ce09dcb57284b, entries=150, sequenceid=82, filesize=11.7 K 2024-11-26T16:41:20,216 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 974fac4d1c1c1475caaca19099f1def5 in 254ms, sequenceid=82, compaction requested=false 2024-11-26T16:41:20,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:20,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:20,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-26T16:41:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-26T16:41:20,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-26T16:41:20,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 563 msec 2024-11-26T16:41:20,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 569 msec 2024-11-26T16:41:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-26T16:41:20,281 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:20,282 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:41:20,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-26T16:41:20,286 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:41:20,287 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:41:20,287 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:41:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T16:41:20,344 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/cd61b7f25ed44987973d4aca55b20b9b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/cd61b7f25ed44987973d4aca55b20b9b 2024-11-26T16:41:20,344 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/944f5ffd08ff4ebcaefd1b80f3e81273 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/944f5ffd08ff4ebcaefd1b80f3e81273 2024-11-26T16:41:20,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 974fac4d1c1c1475caaca19099f1def5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T16:41:20,354 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/A of 974fac4d1c1c1475caaca19099f1def5 into cd61b7f25ed44987973d4aca55b20b9b(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:20,354 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:20,354 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/A, priority=13, startTime=1732639279825; duration=0sec 2024-11-26T16:41:20,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=A 2024-11-26T16:41:20,354 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:20,354 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:A 2024-11-26T16:41:20,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:20,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=B 2024-11-26T16:41:20,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:20,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=C 2024-11-26T16:41:20,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:20,355 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/C of 974fac4d1c1c1475caaca19099f1def5 into 944f5ffd08ff4ebcaefd1b80f3e81273(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:20,355 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:20,355 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/C, priority=13, startTime=1732639279826; duration=0sec 2024-11-26T16:41:20,355 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:20,355 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:C 2024-11-26T16:41:20,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267c0e7f8bef5f47018a133dc6b497adc0_974fac4d1c1c1475caaca19099f1def5 is 50, key is test_row_0/A:col10/1732639280024/Put/seqid=0 2024-11-26T16:41:20,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741971_1147 (size=14594) 2024-11-26T16:41:20,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639340379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639340379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T16:41:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639340384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639340385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639340386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T16:41:20,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:20,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:20,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:20,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:20,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:20,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639340492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639340492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639340501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639340501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639340502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T16:41:20,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:20,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:20,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:20,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:20,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:20,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T16:41:20,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639340712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639340712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639340712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639340713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:20,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639340713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:20,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:20,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T16:41:20,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:20,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:20,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:20,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:20,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:20,775 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:20,782 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267c0e7f8bef5f47018a133dc6b497adc0_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267c0e7f8bef5f47018a133dc6b497adc0_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:20,784 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/4a783db0c5234091bf392ff29e230f23, store: [table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:20,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/4a783db0c5234091bf392ff29e230f23 is 175, key is test_row_0/A:col10/1732639280024/Put/seqid=0 2024-11-26T16:41:20,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741972_1148 (size=39549) 2024-11-26T16:41:20,807 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/4a783db0c5234091bf392ff29e230f23 2024-11-26T16:41:20,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/753426f2e87746f89ad326c4f4a227cc is 50, key is test_row_0/B:col10/1732639280024/Put/seqid=0 2024-11-26T16:41:20,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741973_1149 (size=12001) 2024-11-26T16:41:20,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/753426f2e87746f89ad326c4f4a227cc 2024-11-26T16:41:20,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/3915008a8ac342cbb9b61b6fd3cd0662 is 50, key is test_row_0/C:col10/1732639280024/Put/seqid=0 2024-11-26T16:41:20,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741974_1150 (size=12001) 2024-11-26T16:41:20,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T16:41:20,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:20,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:20,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:20,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:20,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T16:41:21,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639341021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:21,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:21,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639341021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:21,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639341022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:21,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:21,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639341022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:21,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:21,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639341023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:21,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T16:41:21,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:21,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:21,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:21,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:21,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:21,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T16:41:21,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:21,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:21,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:21,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:21,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:21,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/3915008a8ac342cbb9b61b6fd3cd0662 2024-11-26T16:41:21,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/4a783db0c5234091bf392ff29e230f23 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/4a783db0c5234091bf392ff29e230f23 2024-11-26T16:41:21,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/4a783db0c5234091bf392ff29e230f23, entries=200, sequenceid=97, filesize=38.6 K 2024-11-26T16:41:21,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/753426f2e87746f89ad326c4f4a227cc as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/753426f2e87746f89ad326c4f4a227cc 2024-11-26T16:41:21,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/753426f2e87746f89ad326c4f4a227cc, entries=150, sequenceid=97, filesize=11.7 K 2024-11-26T16:41:21,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/3915008a8ac342cbb9b61b6fd3cd0662 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/3915008a8ac342cbb9b61b6fd3cd0662 2024-11-26T16:41:21,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/3915008a8ac342cbb9b61b6fd3cd0662, entries=150, sequenceid=97, filesize=11.7 K 2024-11-26T16:41:21,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 974fac4d1c1c1475caaca19099f1def5 in 927ms, sequenceid=97, compaction requested=true 2024-11-26T16:41:21,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:21,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:41:21,282 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:21,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:21,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:41:21,282 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:21,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:21,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:41:21,284 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:21,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:21,285 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/A is initiating minor compaction (all files) 2024-11-26T16:41:21,285 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/A in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:21,285 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/cd61b7f25ed44987973d4aca55b20b9b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/cc30350b91c74d1dbf1b32296c42c05f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/4a783db0c5234091bf392ff29e230f23] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=99.2 K 2024-11-26T16:41:21,285 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:21,285 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. files: [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/cd61b7f25ed44987973d4aca55b20b9b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/cc30350b91c74d1dbf1b32296c42c05f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/4a783db0c5234091bf392ff29e230f23] 2024-11-26T16:41:21,285 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:21,285 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/B is initiating minor compaction (all files) 2024-11-26T16:41:21,285 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/B in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:21,286 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/a0709759d30f4cc381537334e699b8d6, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/adf93014f9034c668b4095b8636e2060, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/753426f2e87746f89ad326c4f4a227cc] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=35.3 K 2024-11-26T16:41:21,286 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd61b7f25ed44987973d4aca55b20b9b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732639278990 2024-11-26T16:41:21,286 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a0709759d30f4cc381537334e699b8d6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732639278990 2024-11-26T16:41:21,286 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc30350b91c74d1dbf1b32296c42c05f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732639279669 2024-11-26T16:41:21,287 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting adf93014f9034c668b4095b8636e2060, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732639279669 2024-11-26T16:41:21,287 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a783db0c5234091bf392ff29e230f23, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732639280024 2024-11-26T16:41:21,287 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 753426f2e87746f89ad326c4f4a227cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732639280024 2024-11-26T16:41:21,297 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:21,298 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#B#compaction#129 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:21,299 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/073697526a33413fa9ee7f8d367951a0 is 50, key is test_row_0/B:col10/1732639280024/Put/seqid=0 2024-11-26T16:41:21,299 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411269d566c8a78be4212911b51badb45582c_974fac4d1c1c1475caaca19099f1def5 store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:21,303 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411269d566c8a78be4212911b51badb45582c_974fac4d1c1c1475caaca19099f1def5, store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:21,303 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411269d566c8a78be4212911b51badb45582c_974fac4d1c1c1475caaca19099f1def5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:21,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741975_1151 (size=12207) 2024-11-26T16:41:21,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741976_1152 (size=4469) 2024-11-26T16:41:21,314 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#A#compaction#130 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:21,315 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/1955a391c41d41e883b9132bd374a770 is 175, key is test_row_0/A:col10/1732639280024/Put/seqid=0 2024-11-26T16:41:21,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741977_1153 (size=31161) 2024-11-26T16:41:21,330 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/1955a391c41d41e883b9132bd374a770 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/1955a391c41d41e883b9132bd374a770 2024-11-26T16:41:21,337 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/A of 974fac4d1c1c1475caaca19099f1def5 into 1955a391c41d41e883b9132bd374a770(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:21,337 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:21,337 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/A, priority=13, startTime=1732639281282; duration=0sec 2024-11-26T16:41:21,337 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:21,337 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:A 2024-11-26T16:41:21,337 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:21,340 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:21,340 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/C is initiating minor compaction (all files) 2024-11-26T16:41:21,340 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/C in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:21,340 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/944f5ffd08ff4ebcaefd1b80f3e81273, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/80b5e407c2ba4006844ce09dcb57284b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/3915008a8ac342cbb9b61b6fd3cd0662] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=35.3 K 2024-11-26T16:41:21,341 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 944f5ffd08ff4ebcaefd1b80f3e81273, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732639278990 2024-11-26T16:41:21,341 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80b5e407c2ba4006844ce09dcb57284b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732639279669 2024-11-26T16:41:21,342 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3915008a8ac342cbb9b61b6fd3cd0662, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732639280024 2024-11-26T16:41:21,350 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#C#compaction#131 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:21,350 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/01a870d7328b420da79fbc4dfd12292f is 50, key is test_row_0/C:col10/1732639280024/Put/seqid=0 2024-11-26T16:41:21,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741978_1154 (size=12207) 2024-11-26T16:41:21,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-26T16:41:21,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:21,368 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 974fac4d1c1c1475caaca19099f1def5 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T16:41:21,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=A 2024-11-26T16:41:21,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:21,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=B 2024-11-26T16:41:21,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:21,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=C 2024-11-26T16:41:21,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:21,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b242bd13d8374d08926bbfa88d9c5cf1_974fac4d1c1c1475caaca19099f1def5 is 50, key is test_row_0/A:col10/1732639280384/Put/seqid=0 2024-11-26T16:41:21,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741979_1155 (size=12154) 2024-11-26T16:41:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T16:41:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:21,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:21,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639341545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:21,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639341546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:21,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639341547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:21,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639341548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:21,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:21,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639341554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:21,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:21,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639341661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:21,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639341661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:21,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:21,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:21,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:21,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639341671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639341671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:21,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639341671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:21,720 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/073697526a33413fa9ee7f8d367951a0 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/073697526a33413fa9ee7f8d367951a0 2024-11-26T16:41:21,727 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/B of 974fac4d1c1c1475caaca19099f1def5 into 073697526a33413fa9ee7f8d367951a0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:21,727 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:21,727 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/B, priority=13, startTime=1732639281282; duration=0sec 2024-11-26T16:41:21,727 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:21,727 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:B 2024-11-26T16:41:21,767 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/01a870d7328b420da79fbc4dfd12292f as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/01a870d7328b420da79fbc4dfd12292f 2024-11-26T16:41:21,774 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/C of 974fac4d1c1c1475caaca19099f1def5 into 01a870d7328b420da79fbc4dfd12292f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:21,774 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:21,774 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/C, priority=13, startTime=1732639281284; duration=0sec 2024-11-26T16:41:21,774 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:21,774 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:C 2024-11-26T16:41:21,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:21,788 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b242bd13d8374d08926bbfa88d9c5cf1_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b242bd13d8374d08926bbfa88d9c5cf1_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:21,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/7bcb33f6abd041deb0a3c0934463d70a, store: [table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:21,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/7bcb33f6abd041deb0a3c0934463d70a is 175, key is test_row_0/A:col10/1732639280384/Put/seqid=0 2024-11-26T16:41:21,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741980_1156 (size=30955) 2024-11-26T16:41:21,856 INFO [master/2dedfe7466e8:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-26T16:41:21,856 INFO [master/2dedfe7466e8:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-26T16:41:21,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:21,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639341871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639341871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:21,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:21,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:21,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639341882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:21,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:21,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639341891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:21,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639341891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:22,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639342182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:22,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639342182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:22,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639342192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:22,203 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/7bcb33f6abd041deb0a3c0934463d70a 2024-11-26T16:41:22,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639342202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:22,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639342203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:22,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:22,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:22,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:22,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/7073103a63184f0b9c7b166a68a2c5b5 is 50, key is test_row_0/B:col10/1732639280384/Put/seqid=0 2024-11-26T16:41:22,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741981_1157 (size=12001) 2024-11-26T16:41:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T16:41:22,630 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/7073103a63184f0b9c7b166a68a2c5b5 2024-11-26T16:41:22,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/68c9a2395ebb4be8a94f52390df0bd00 is 50, key is test_row_0/C:col10/1732639280384/Put/seqid=0 2024-11-26T16:41:22,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741982_1158 (size=12001) 2024-11-26T16:41:22,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639342691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:22,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:22,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:22,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:22,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639342701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:22,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:22,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639342711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:22,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639342711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:22,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:22,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:22,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:22,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:22,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639342711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:22,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:22,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:22,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:22,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:22,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:23,046 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/68c9a2395ebb4be8a94f52390df0bd00 2024-11-26T16:41:23,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/7bcb33f6abd041deb0a3c0934463d70a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/7bcb33f6abd041deb0a3c0934463d70a 2024-11-26T16:41:23,058 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/7bcb33f6abd041deb0a3c0934463d70a, entries=150, sequenceid=117, filesize=30.2 K 2024-11-26T16:41:23,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/7073103a63184f0b9c7b166a68a2c5b5 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/7073103a63184f0b9c7b166a68a2c5b5 2024-11-26T16:41:23,065 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/7073103a63184f0b9c7b166a68a2c5b5, entries=150, sequenceid=117, filesize=11.7 K 2024-11-26T16:41:23,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/68c9a2395ebb4be8a94f52390df0bd00 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/68c9a2395ebb4be8a94f52390df0bd00 2024-11-26T16:41:23,071 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/68c9a2395ebb4be8a94f52390df0bd00, entries=150, sequenceid=117, filesize=11.7 K 2024-11-26T16:41:23,072 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 974fac4d1c1c1475caaca19099f1def5 in 1704ms, sequenceid=117, compaction requested=false 2024-11-26T16:41:23,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:23,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:23,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-26T16:41:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-26T16:41:23,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-26T16:41:23,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7870 sec 2024-11-26T16:41:23,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.7940 sec 2024-11-26T16:41:23,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 974fac4d1c1c1475caaca19099f1def5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T16:41:23,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=A 2024-11-26T16:41:23,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:23,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=B 2024-11-26T16:41:23,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:23,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=C 2024-11-26T16:41:23,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:23,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f243f76cbb3a41b2beff9efdc6a89cf4_974fac4d1c1c1475caaca19099f1def5 is 50, key is test_row_0/A:col10/1732639281551/Put/seqid=0 2024-11-26T16:41:23,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741983_1159 (size=14694) 2024-11-26T16:41:23,751 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:23,757 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f243f76cbb3a41b2beff9efdc6a89cf4_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f243f76cbb3a41b2beff9efdc6a89cf4_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:23,764 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/8d2c3ba9aa36458fa48610b541d511c1, store: [table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:23,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/8d2c3ba9aa36458fa48610b541d511c1 is 175, key is test_row_0/A:col10/1732639281551/Put/seqid=0 2024-11-26T16:41:23,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639343756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:23,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639343765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:23,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:23,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:23,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:23,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639343768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:23,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741984_1160 (size=39649) 2024-11-26T16:41:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:23,780 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/8d2c3ba9aa36458fa48610b541d511c1 2024-11-26T16:41:23,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639343772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:23,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:23,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639343772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:23,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:23,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:23,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:23,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:23,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:23,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/42331f0fce6646239a157c5f54018e1a is 50, key is test_row_0/B:col10/1732639281551/Put/seqid=0 2024-11-26T16:41:23,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741985_1161 (size=12101) 2024-11-26T16:41:23,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639343881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:23,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:23,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:23,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639343881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:23,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:23,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639343881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:23,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:23,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:23,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:23,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:23,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:23,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:23,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:23,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639343891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:23,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:23,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639343891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:23,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:23,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:23,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:23,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:23,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639344091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:24,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:24,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639344091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:24,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639344091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639344101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:24,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639344102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/42331f0fce6646239a157c5f54018e1a 2024-11-26T16:41:24,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/149846a986a2431ba298c966e2311cbd is 50, key is test_row_0/C:col10/1732639281551/Put/seqid=0 2024-11-26T16:41:24,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741986_1162 (size=12101) 2024-11-26T16:41:24,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/149846a986a2431ba298c966e2311cbd 2024-11-26T16:41:24,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/8d2c3ba9aa36458fa48610b541d511c1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/8d2c3ba9aa36458fa48610b541d511c1 2024-11-26T16:41:24,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/8d2c3ba9aa36458fa48610b541d511c1, entries=200, sequenceid=134, filesize=38.7 K 2024-11-26T16:41:24,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/42331f0fce6646239a157c5f54018e1a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/42331f0fce6646239a157c5f54018e1a 2024-11-26T16:41:24,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/42331f0fce6646239a157c5f54018e1a, entries=150, sequenceid=134, filesize=11.8 K 2024-11-26T16:41:24,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/149846a986a2431ba298c966e2311cbd as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/149846a986a2431ba298c966e2311cbd 2024-11-26T16:41:24,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/149846a986a2431ba298c966e2311cbd, entries=150, sequenceid=134, filesize=11.8 K 2024-11-26T16:41:24,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 974fac4d1c1c1475caaca19099f1def5 in 541ms, sequenceid=134, compaction requested=true 2024-11-26T16:41:24,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:24,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:41:24,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:24,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:41:24,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:24,247 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:24,247 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:24,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:41:24,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:24,248 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101765 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:24,248 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/A is initiating minor compaction (all files) 2024-11-26T16:41:24,248 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/A in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:24,248 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/1955a391c41d41e883b9132bd374a770, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/7bcb33f6abd041deb0a3c0934463d70a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/8d2c3ba9aa36458fa48610b541d511c1] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=99.4 K 2024-11-26T16:41:24,248 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:24,248 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. files: [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/1955a391c41d41e883b9132bd374a770, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/7bcb33f6abd041deb0a3c0934463d70a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/8d2c3ba9aa36458fa48610b541d511c1] 2024-11-26T16:41:24,249 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:24,249 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/B is initiating minor compaction (all files) 2024-11-26T16:41:24,249 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/B in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:24,249 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/073697526a33413fa9ee7f8d367951a0, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/7073103a63184f0b9c7b166a68a2c5b5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/42331f0fce6646239a157c5f54018e1a] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=35.5 K 2024-11-26T16:41:24,250 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1955a391c41d41e883b9132bd374a770, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732639280024 2024-11-26T16:41:24,250 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 073697526a33413fa9ee7f8d367951a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732639280024 2024-11-26T16:41:24,250 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bcb33f6abd041deb0a3c0934463d70a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732639280379 2024-11-26T16:41:24,251 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 7073103a63184f0b9c7b166a68a2c5b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732639280379 2024-11-26T16:41:24,251 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d2c3ba9aa36458fa48610b541d511c1, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732639281537 2024-11-26T16:41:24,251 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 42331f0fce6646239a157c5f54018e1a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732639281544 2024-11-26T16:41:24,270 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:24,271 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/1b5c0401812d427da4de2a27512aab20 is 50, key is test_row_0/B:col10/1732639281551/Put/seqid=0 2024-11-26T16:41:24,276 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:24,302 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411261e647748b3cf46c396f85b353c5663db_974fac4d1c1c1475caaca19099f1def5 store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:24,305 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411261e647748b3cf46c396f85b353c5663db_974fac4d1c1c1475caaca19099f1def5, store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:24,305 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261e647748b3cf46c396f85b353c5663db_974fac4d1c1c1475caaca19099f1def5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:24,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741987_1163 (size=12409) 2024-11-26T16:41:24,321 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/1b5c0401812d427da4de2a27512aab20 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/1b5c0401812d427da4de2a27512aab20 2024-11-26T16:41:24,332 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/B of 974fac4d1c1c1475caaca19099f1def5 into 1b5c0401812d427da4de2a27512aab20(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:24,332 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:24,332 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/B, priority=13, startTime=1732639284247; duration=0sec 2024-11-26T16:41:24,333 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:24,333 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:B 2024-11-26T16:41:24,333 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:24,336 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:24,336 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/C is initiating minor compaction (all files) 2024-11-26T16:41:24,337 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/C in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:24,337 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/01a870d7328b420da79fbc4dfd12292f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/68c9a2395ebb4be8a94f52390df0bd00, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/149846a986a2431ba298c966e2311cbd] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=35.5 K 2024-11-26T16:41:24,337 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 01a870d7328b420da79fbc4dfd12292f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732639280024 2024-11-26T16:41:24,338 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 68c9a2395ebb4be8a94f52390df0bd00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732639280379 2024-11-26T16:41:24,338 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 149846a986a2431ba298c966e2311cbd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732639281544 2024-11-26T16:41:24,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741988_1164 (size=4469) 2024-11-26T16:41:24,348 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#A#compaction#139 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:24,349 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/95d508ca20434af2987f4364110f5a71 is 175, key is test_row_0/A:col10/1732639281551/Put/seqid=0 2024-11-26T16:41:24,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741989_1165 (size=31363) 2024-11-26T16:41:24,367 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#C#compaction#140 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:24,368 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/a307b24633a34f7f8993f90aee05ac25 is 50, key is test_row_0/C:col10/1732639281551/Put/seqid=0 2024-11-26T16:41:24,375 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/95d508ca20434af2987f4364110f5a71 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/95d508ca20434af2987f4364110f5a71 2024-11-26T16:41:24,382 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/A of 974fac4d1c1c1475caaca19099f1def5 into 95d508ca20434af2987f4364110f5a71(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:24,382 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:24,383 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/A, priority=13, startTime=1732639284247; duration=0sec 2024-11-26T16:41:24,383 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:24,383 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:A 2024-11-26T16:41:24,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741990_1166 (size=12409) 2024-11-26T16:41:24,402 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/a307b24633a34f7f8993f90aee05ac25 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/a307b24633a34f7f8993f90aee05ac25 2024-11-26T16:41:24,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 974fac4d1c1c1475caaca19099f1def5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T16:41:24,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=A 2024-11-26T16:41:24,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:24,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=B 2024-11-26T16:41:24,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:24,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=C 2024-11-26T16:41:24,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:24,410 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/C of 974fac4d1c1c1475caaca19099f1def5 into a307b24633a34f7f8993f90aee05ac25(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:24,410 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:24,410 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/C, priority=13, startTime=1732639284247; duration=0sec 2024-11-26T16:41:24,410 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:24,410 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:C 2024-11-26T16:41:24,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126cfa882ee9ef44177be2148ecd831a8b3_974fac4d1c1c1475caaca19099f1def5 is 50, key is test_row_0/A:col10/1732639283765/Put/seqid=0 2024-11-26T16:41:24,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639344440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639344445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:24,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639344446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639344443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:24,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741991_1167 (size=17284) 2024-11-26T16:41:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-26T16:41:24,462 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:24,462 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:41:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-26T16:41:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T16:41:24,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639344454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:24,466 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:41:24,467 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:41:24,467 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:41:24,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639344561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:24,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639344561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639344561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639344561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T16:41:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639344572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T16:41:24,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:24,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:24,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:24,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:24,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:24,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T16:41:24,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:24,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:24,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:24,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:24,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T16:41:24,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639344781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:24,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639344781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:24,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639344781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639344782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639344782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:24,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:24,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:24,860 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:24,865 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126cfa882ee9ef44177be2148ecd831a8b3_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cfa882ee9ef44177be2148ecd831a8b3_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:24,866 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/8a8b5954a0d24e7db7c0ff4571d59bab, store: [table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:24,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/8a8b5954a0d24e7db7c0ff4571d59bab is 175, key is test_row_0/A:col10/1732639283765/Put/seqid=0 2024-11-26T16:41:24,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741992_1168 (size=48389) 2024-11-26T16:41:24,872 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/8a8b5954a0d24e7db7c0ff4571d59bab 2024-11-26T16:41:24,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/03eb89e55df0425fae9111794b8efc3e is 50, key is test_row_0/B:col10/1732639283765/Put/seqid=0 2024-11-26T16:41:24,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741993_1169 (size=12151) 2024-11-26T16:41:24,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/03eb89e55df0425fae9111794b8efc3e 2024-11-26T16:41:24,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/695ba50dcc404791910630688d018f2c is 50, key is test_row_0/C:col10/1732639283765/Put/seqid=0 2024-11-26T16:41:24,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741994_1170 (size=12151) 2024-11-26T16:41:24,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T16:41:24,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:24,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:24,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:24,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:24,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:25,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T16:41:25,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:25,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:25,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:25,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:25,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T16:41:25,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:25,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639345091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:25,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:25,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:25,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639345100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:25,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:25,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639345101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:25,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:25,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:25,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:25,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639345101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:25,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:25,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:25,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639345101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:25,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T16:41:25,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:25,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:25,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:25,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:25,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:25,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:25,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/695ba50dcc404791910630688d018f2c 2024-11-26T16:41:25,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/8a8b5954a0d24e7db7c0ff4571d59bab as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/8a8b5954a0d24e7db7c0ff4571d59bab 2024-11-26T16:41:25,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/8a8b5954a0d24e7db7c0ff4571d59bab, entries=250, sequenceid=156, filesize=47.3 K 2024-11-26T16:41:25,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/03eb89e55df0425fae9111794b8efc3e as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/03eb89e55df0425fae9111794b8efc3e 2024-11-26T16:41:25,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/03eb89e55df0425fae9111794b8efc3e, entries=150, sequenceid=156, filesize=11.9 K 2024-11-26T16:41:25,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/695ba50dcc404791910630688d018f2c as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/695ba50dcc404791910630688d018f2c 2024-11-26T16:41:25,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/695ba50dcc404791910630688d018f2c, entries=150, sequenceid=156, filesize=11.9 K 2024-11-26T16:41:25,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 974fac4d1c1c1475caaca19099f1def5 in 926ms, sequenceid=156, compaction requested=false 2024-11-26T16:41:25,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:25,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-26T16:41:25,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:25,388 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 974fac4d1c1c1475caaca19099f1def5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T16:41:25,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=A 2024-11-26T16:41:25,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:25,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=B 2024-11-26T16:41:25,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:25,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=C 2024-11-26T16:41:25,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:25,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126094043e5b5de4e5cb480849c19ea7512_974fac4d1c1c1475caaca19099f1def5 is 50, key is test_row_0/A:col10/1732639284440/Put/seqid=0 2024-11-26T16:41:25,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741995_1171 (size=12304) 2024-11-26T16:41:25,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:25,411 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126094043e5b5de4e5cb480849c19ea7512_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126094043e5b5de4e5cb480849c19ea7512_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:25,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/51a610a45df540aa81c4b00fc35f92a2, store: [table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:25,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/51a610a45df540aa81c4b00fc35f92a2 is 175, key is test_row_0/A:col10/1732639284440/Put/seqid=0 2024-11-26T16:41:25,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741996_1172 (size=31105) 2024-11-26T16:41:25,418 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/51a610a45df540aa81c4b00fc35f92a2 2024-11-26T16:41:25,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/d9feef9f0bf348a09d3d90d6424b02fa is 50, key is test_row_0/B:col10/1732639284440/Put/seqid=0 2024-11-26T16:41:25,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741997_1173 (size=12151) 2024-11-26T16:41:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T16:41:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:25,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:25,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:25,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639345630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:25,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:25,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639345632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:25,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639345632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:25,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639345633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:25,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639345633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:25,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639345741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:25,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639345742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:25,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639345742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:25,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639345751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:25,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:25,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639345751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:25,843 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/d9feef9f0bf348a09d3d90d6424b02fa 2024-11-26T16:41:25,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/90851ecf66c9487eb81d8d08bcd52fe4 is 50, key is test_row_0/C:col10/1732639284440/Put/seqid=0 2024-11-26T16:41:25,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741998_1174 (size=12151) 2024-11-26T16:41:25,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:25,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639345951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639345951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:25,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639345951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:25,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:25,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639345961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:25,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:25,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:25,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639345961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:25,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:25,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:26,258 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/90851ecf66c9487eb81d8d08bcd52fe4 2024-11-26T16:41:26,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639346261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:26,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:26,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639346261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:26,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:26,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:26,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639346261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:26,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/51a610a45df540aa81c4b00fc35f92a2 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/51a610a45df540aa81c4b00fc35f92a2 2024-11-26T16:41:26,269 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/51a610a45df540aa81c4b00fc35f92a2, entries=150, sequenceid=170, filesize=30.4 K 2024-11-26T16:41:26,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/d9feef9f0bf348a09d3d90d6424b02fa as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/d9feef9f0bf348a09d3d90d6424b02fa 2024-11-26T16:41:26,275 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/d9feef9f0bf348a09d3d90d6424b02fa, entries=150, sequenceid=170, filesize=11.9 K 2024-11-26T16:41:26,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639346271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:26,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639346271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:26,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/90851ecf66c9487eb81d8d08bcd52fe4 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/90851ecf66c9487eb81d8d08bcd52fe4 2024-11-26T16:41:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:26,282 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/90851ecf66c9487eb81d8d08bcd52fe4, entries=150, sequenceid=170, filesize=11.9 K 2024-11-26T16:41:26,283 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 974fac4d1c1c1475caaca19099f1def5 in 895ms, sequenceid=170, compaction requested=true 2024-11-26T16:41:26,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:26,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:26,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-26T16:41:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-26T16:41:26,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-26T16:41:26,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8180 sec 2024-11-26T16:41:26,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.8250 sec 2024-11-26T16:41:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-26T16:41:26,612 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:26,612 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:41:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-26T16:41:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T16:41:26,616 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:41:26,617 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:41:26,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:41:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T16:41:26,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-26T16:41:26,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:26,770 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 974fac4d1c1c1475caaca19099f1def5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T16:41:26,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=A 2024-11-26T16:41:26,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:26,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=B 2024-11-26T16:41:26,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:26,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=C 2024-11-26T16:41:26,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:26,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:26,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126438db52ab3a24095bdf2085278406d51_974fac4d1c1c1475caaca19099f1def5 is 50, key is test_row_0/A:col10/1732639285631/Put/seqid=0 2024-11-26T16:41:26,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639346785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:26,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639346786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:26,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639346791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:26,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639346795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:26,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639346797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:26,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741999_1175 (size=12304) 2024-11-26T16:41:26,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:26,824 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126438db52ab3a24095bdf2085278406d51_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126438db52ab3a24095bdf2085278406d51_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:26,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/0ef3a3851c9142c6975507058cc4732f, store: [table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:26,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/0ef3a3851c9142c6975507058cc4732f is 175, key is test_row_0/A:col10/1732639285631/Put/seqid=0 2024-11-26T16:41:26,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742000_1176 (size=31105) 2024-11-26T16:41:26,842 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/0ef3a3851c9142c6975507058cc4732f 2024-11-26T16:41:26,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/f9da74b279ce4ca7a0b13e4daf91865c is 50, key is test_row_0/B:col10/1732639285631/Put/seqid=0 2024-11-26T16:41:26,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742001_1177 (size=12151) 2024-11-26T16:41:26,862 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/f9da74b279ce4ca7a0b13e4daf91865c 2024-11-26T16:41:26,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/3f4aff44ab54421c8d8d4b0632446861 is 50, key is test_row_0/C:col10/1732639285631/Put/seqid=0 2024-11-26T16:41:26,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742002_1178 (size=12151) 2024-11-26T16:41:26,881 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/3f4aff44ab54421c8d8d4b0632446861 2024-11-26T16:41:26,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/0ef3a3851c9142c6975507058cc4732f as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/0ef3a3851c9142c6975507058cc4732f 2024-11-26T16:41:26,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639346901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:26,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639346901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:26,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:26,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:26,911 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/0ef3a3851c9142c6975507058cc4732f, entries=150, sequenceid=190, filesize=30.4 K 2024-11-26T16:41:26,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/f9da74b279ce4ca7a0b13e4daf91865c as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/f9da74b279ce4ca7a0b13e4daf91865c 2024-11-26T16:41:26,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639346911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:26,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:26,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639346911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:26,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:26,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:26,920 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/f9da74b279ce4ca7a0b13e4daf91865c, entries=150, sequenceid=190, filesize=11.9 K 2024-11-26T16:41:26,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/3f4aff44ab54421c8d8d4b0632446861 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/3f4aff44ab54421c8d8d4b0632446861 2024-11-26T16:41:26,928 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/3f4aff44ab54421c8d8d4b0632446861, entries=150, sequenceid=190, filesize=11.9 K 2024-11-26T16:41:26,929 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 974fac4d1c1c1475caaca19099f1def5 in 159ms, sequenceid=190, compaction requested=true 2024-11-26T16:41:26,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:26,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:26,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-26T16:41:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-26T16:41:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T16:41:26,934 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-26T16:41:26,934 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 314 msec 2024-11-26T16:41:26,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 323 msec 2024-11-26T16:41:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:27,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 974fac4d1c1c1475caaca19099f1def5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T16:41:27,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=A 2024-11-26T16:41:27,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:27,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=B 2024-11-26T16:41:27,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:27,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=C 2024-11-26T16:41:27,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:27,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126249e34dce5d94f899ae2824da744f09e_974fac4d1c1c1475caaca19099f1def5 is 50, key is test_row_0/A:col10/1732639286795/Put/seqid=0 2024-11-26T16:41:27,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742003_1179 (size=14794) 2024-11-26T16:41:27,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639347143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:27,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:27,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639347147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:27,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639347149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:27,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:27,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:27,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639347150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:27,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-26T16:41:27,241 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:27,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:41:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-26T16:41:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T16:41:27,244 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:41:27,244 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:41:27,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:41:27,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639347261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:27,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:27,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639347261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:27,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639347261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639347261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T16:41:27,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T16:41:27,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:27,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:27,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:27,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:27,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:27,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639347481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:27,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:27,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639347481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:27,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639347482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:27,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639347482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:27,530 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:27,535 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126249e34dce5d94f899ae2824da744f09e_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126249e34dce5d94f899ae2824da744f09e_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:27,536 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/c8450e1dbecc4be7b4b4d758c3c0b521, store: [table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:27,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/c8450e1dbecc4be7b4b4d758c3c0b521 is 175, key is test_row_0/A:col10/1732639286795/Put/seqid=0 2024-11-26T16:41:27,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742004_1180 (size=39749) 2024-11-26T16:41:27,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T16:41:27,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:27,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:27,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:27,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:27,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T16:41:27,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T16:41:27,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:27,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:27,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:27,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:27,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:27,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639347801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:27,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639347802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:27,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639347802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639347802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:27,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639347811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:27,814 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in 974fac4d1c1c1475caaca19099f1def5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:41:27,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:27,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:27,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T16:41:27,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:27,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:27,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:27,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:27,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T16:41:27,942 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/c8450e1dbecc4be7b4b4d758c3c0b521 2024-11-26T16:41:27,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/f16fca1f19ba459797bab065b7df1609 is 50, key is test_row_0/B:col10/1732639286795/Put/seqid=0 2024-11-26T16:41:27,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742005_1181 (size=12151) 2024-11-26T16:41:28,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T16:41:28,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:28,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:28,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:28,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:28,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:28,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T16:41:28,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:28,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:28,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:28,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:28,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:28,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639348311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:28,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:28,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639348311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:28,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:28,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:28,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:28,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639348312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:28,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639348312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:28,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T16:41:28,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:28,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:28,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:28,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:28,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:28,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/f16fca1f19ba459797bab065b7df1609 2024-11-26T16:41:28,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/04dabd59eb214d698fed0626cd015286 is 50, key is test_row_0/C:col10/1732639286795/Put/seqid=0 2024-11-26T16:41:28,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742006_1182 (size=12151) 2024-11-26T16:41:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T16:41:28,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T16:41:28,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:28,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:28,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:28,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:28,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:28,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T16:41:28,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:28,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:28,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:28,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:28,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:28,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/04dabd59eb214d698fed0626cd015286 2024-11-26T16:41:28,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/c8450e1dbecc4be7b4b4d758c3c0b521 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/c8450e1dbecc4be7b4b4d758c3c0b521 2024-11-26T16:41:28,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T16:41:28,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:28,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:28,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:28,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:28,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:28,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/c8450e1dbecc4be7b4b4d758c3c0b521, entries=200, sequenceid=203, filesize=38.8 K 2024-11-26T16:41:28,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/f16fca1f19ba459797bab065b7df1609 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/f16fca1f19ba459797bab065b7df1609 2024-11-26T16:41:28,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/f16fca1f19ba459797bab065b7df1609, entries=150, sequenceid=203, filesize=11.9 K 2024-11-26T16:41:28,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/04dabd59eb214d698fed0626cd015286 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/04dabd59eb214d698fed0626cd015286 2024-11-26T16:41:28,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/04dabd59eb214d698fed0626cd015286, entries=150, sequenceid=203, filesize=11.9 K 2024-11-26T16:41:28,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 974fac4d1c1c1475caaca19099f1def5 in 1677ms, sequenceid=203, compaction requested=true 2024-11-26T16:41:28,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:28,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:41:28,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:28,793 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-26T16:41:28,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:41:28,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:28,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:41:28,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:28,793 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-26T16:41:28,799 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61013 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-26T16:41:28,799 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/B is initiating minor compaction (all files) 2024-11-26T16:41:28,799 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/B in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:28,799 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 181711 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-26T16:41:28,799 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/A is initiating minor compaction (all files) 2024-11-26T16:41:28,799 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/1b5c0401812d427da4de2a27512aab20, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/03eb89e55df0425fae9111794b8efc3e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/d9feef9f0bf348a09d3d90d6424b02fa, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/f9da74b279ce4ca7a0b13e4daf91865c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/f16fca1f19ba459797bab065b7df1609] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=59.6 K 2024-11-26T16:41:28,799 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/A in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:28,799 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/95d508ca20434af2987f4364110f5a71, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/8a8b5954a0d24e7db7c0ff4571d59bab, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/51a610a45df540aa81c4b00fc35f92a2, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/0ef3a3851c9142c6975507058cc4732f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/c8450e1dbecc4be7b4b4d758c3c0b521] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=177.5 K 2024-11-26T16:41:28,799 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:28,799 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. files: [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/95d508ca20434af2987f4364110f5a71, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/8a8b5954a0d24e7db7c0ff4571d59bab, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/51a610a45df540aa81c4b00fc35f92a2, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/0ef3a3851c9142c6975507058cc4732f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/c8450e1dbecc4be7b4b4d758c3c0b521] 2024-11-26T16:41:28,799 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b5c0401812d427da4de2a27512aab20, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732639281544 2024-11-26T16:41:28,800 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95d508ca20434af2987f4364110f5a71, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732639281544 2024-11-26T16:41:28,800 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 03eb89e55df0425fae9111794b8efc3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732639283763 2024-11-26T16:41:28,800 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a8b5954a0d24e7db7c0ff4571d59bab, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732639283763 2024-11-26T16:41:28,800 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting d9feef9f0bf348a09d3d90d6424b02fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732639284438 2024-11-26T16:41:28,801 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51a610a45df540aa81c4b00fc35f92a2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732639284438 2024-11-26T16:41:28,801 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting f9da74b279ce4ca7a0b13e4daf91865c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732639285620 2024-11-26T16:41:28,801 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ef3a3851c9142c6975507058cc4732f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732639285620 2024-11-26T16:41:28,801 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting f16fca1f19ba459797bab065b7df1609, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732639286784 2024-11-26T16:41:28,801 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8450e1dbecc4be7b4b4d758c3c0b521, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732639286784 2024-11-26T16:41:28,816 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:28,818 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411267b7d8c751bbb4b188e8d46ce3234499b_974fac4d1c1c1475caaca19099f1def5 store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:28,818 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#B#compaction#153 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:28,819 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/94b1cfaf3beb4fb887a6173dac4f7634 is 50, key is test_row_0/B:col10/1732639286795/Put/seqid=0 2024-11-26T16:41:28,821 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411267b7d8c751bbb4b188e8d46ce3234499b_974fac4d1c1c1475caaca19099f1def5, store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:28,821 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267b7d8c751bbb4b188e8d46ce3234499b_974fac4d1c1c1475caaca19099f1def5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:28,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742007_1183 (size=12629) 2024-11-26T16:41:28,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742008_1184 (size=4469) 2024-11-26T16:41:28,832 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/94b1cfaf3beb4fb887a6173dac4f7634 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/94b1cfaf3beb4fb887a6173dac4f7634 2024-11-26T16:41:28,838 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/B of 974fac4d1c1c1475caaca19099f1def5 into 94b1cfaf3beb4fb887a6173dac4f7634(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:28,838 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:28,838 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/B, priority=11, startTime=1732639288793; duration=0sec 2024-11-26T16:41:28,838 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:28,838 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:B 2024-11-26T16:41:28,838 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-26T16:41:28,840 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61013 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-26T16:41:28,840 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/C is initiating minor compaction (all files) 2024-11-26T16:41:28,840 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/C in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:28,841 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/a307b24633a34f7f8993f90aee05ac25, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/695ba50dcc404791910630688d018f2c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/90851ecf66c9487eb81d8d08bcd52fe4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/3f4aff44ab54421c8d8d4b0632446861, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/04dabd59eb214d698fed0626cd015286] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=59.6 K 2024-11-26T16:41:28,841 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a307b24633a34f7f8993f90aee05ac25, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732639281544 2024-11-26T16:41:28,841 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 695ba50dcc404791910630688d018f2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732639283763 2024-11-26T16:41:28,842 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 90851ecf66c9487eb81d8d08bcd52fe4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732639284438 2024-11-26T16:41:28,842 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f4aff44ab54421c8d8d4b0632446861, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732639285620 2024-11-26T16:41:28,843 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 04dabd59eb214d698fed0626cd015286, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732639286784 2024-11-26T16:41:28,855 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#C#compaction#155 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:28,856 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/223284b55a4842b188591499f4c14482 is 50, key is test_row_0/C:col10/1732639286795/Put/seqid=0 2024-11-26T16:41:28,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742009_1185 (size=12629) 2024-11-26T16:41:28,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-26T16:41:28,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:28,933 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 974fac4d1c1c1475caaca19099f1def5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T16:41:28,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=A 2024-11-26T16:41:28,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:28,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=B 2024-11-26T16:41:28,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:28,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=C 2024-11-26T16:41:28,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:28,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126023360b3e4aa46e4bf0d6ad2e1947f59_974fac4d1c1c1475caaca19099f1def5 is 50, key is test_row_0/A:col10/1732639287148/Put/seqid=0 2024-11-26T16:41:28,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742010_1186 (size=12304) 2024-11-26T16:41:28,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:28,952 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126023360b3e4aa46e4bf0d6ad2e1947f59_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126023360b3e4aa46e4bf0d6ad2e1947f59_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:28,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/30a49593e10c495b8c5a92d0513ecf58, store: [table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:28,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/30a49593e10c495b8c5a92d0513ecf58 is 175, key is test_row_0/A:col10/1732639287148/Put/seqid=0 2024-11-26T16:41:28,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742011_1187 (size=31105) 2024-11-26T16:41:29,228 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#A#compaction#154 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:29,229 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/156d57359b154637be31e97fc6a7d63d is 175, key is test_row_0/A:col10/1732639286795/Put/seqid=0 2024-11-26T16:41:29,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742012_1188 (size=31583) 2024-11-26T16:41:29,266 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/223284b55a4842b188591499f4c14482 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/223284b55a4842b188591499f4c14482 2024-11-26T16:41:29,272 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/C of 974fac4d1c1c1475caaca19099f1def5 into 223284b55a4842b188591499f4c14482(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:29,272 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:29,272 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/C, priority=11, startTime=1732639288793; duration=0sec 2024-11-26T16:41:29,272 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:29,272 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:C 2024-11-26T16:41:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:29,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:29,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639349335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:29,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:29,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639349341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:29,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:29,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:29,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639349342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:29,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:29,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639349344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:29,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:29,361 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/30a49593e10c495b8c5a92d0513ecf58 2024-11-26T16:41:29,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/2e97430fd50f433c8fff60102368a222 is 50, key is test_row_0/B:col10/1732639287148/Put/seqid=0 2024-11-26T16:41:29,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742013_1189 (size=12151) 2024-11-26T16:41:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T16:41:29,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639349451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:29,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639349451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:29,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:29,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:29,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639349451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:29,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639349451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:29,639 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/156d57359b154637be31e97fc6a7d63d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/156d57359b154637be31e97fc6a7d63d 2024-11-26T16:41:29,644 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/A of 974fac4d1c1c1475caaca19099f1def5 into 156d57359b154637be31e97fc6a7d63d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:29,644 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:29,644 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/A, priority=11, startTime=1732639288792; duration=0sec 2024-11-26T16:41:29,645 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:29,645 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:A 2024-11-26T16:41:29,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639349661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:29,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:29,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639349661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:29,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:29,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:29,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639349661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:29,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639349661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:29,776 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/2e97430fd50f433c8fff60102368a222 2024-11-26T16:41:29,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/a4695b892d924b86a0478335ea554437 is 50, key is test_row_0/C:col10/1732639287148/Put/seqid=0 2024-11-26T16:41:29,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742014_1190 (size=12151) 2024-11-26T16:41:29,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:29,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639349841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:29,845 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in 974fac4d1c1c1475caaca19099f1def5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:41:29,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:29,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639349971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:29,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:29,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639349972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:29,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:29,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:29,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639349972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:29,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:29,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:29,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639349972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:29,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:29,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:29,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:30,190 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/a4695b892d924b86a0478335ea554437 2024-11-26T16:41:30,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/30a49593e10c495b8c5a92d0513ecf58 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/30a49593e10c495b8c5a92d0513ecf58 2024-11-26T16:41:30,200 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/30a49593e10c495b8c5a92d0513ecf58, entries=150, sequenceid=223, filesize=30.4 K 2024-11-26T16:41:30,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/2e97430fd50f433c8fff60102368a222 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/2e97430fd50f433c8fff60102368a222 2024-11-26T16:41:30,205 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/2e97430fd50f433c8fff60102368a222, entries=150, sequenceid=223, filesize=11.9 K 2024-11-26T16:41:30,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/a4695b892d924b86a0478335ea554437 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/a4695b892d924b86a0478335ea554437 2024-11-26T16:41:30,211 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/a4695b892d924b86a0478335ea554437, entries=150, sequenceid=223, filesize=11.9 K 2024-11-26T16:41:30,212 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 974fac4d1c1c1475caaca19099f1def5 in 1279ms, sequenceid=223, compaction requested=false 2024-11-26T16:41:30,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:30,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:30,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-26T16:41:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-26T16:41:30,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-26T16:41:30,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9690 sec 2024-11-26T16:41:30,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.9750 sec 2024-11-26T16:41:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:30,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 974fac4d1c1c1475caaca19099f1def5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-26T16:41:30,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=A 2024-11-26T16:41:30,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:30,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=B 2024-11-26T16:41:30,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:30,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=C 2024-11-26T16:41:30,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:30,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267fa07e2a6ab5411ebe5e0a516971c4dd_974fac4d1c1c1475caaca19099f1def5 is 50, key is test_row_0/A:col10/1732639290482/Put/seqid=0 2024-11-26T16:41:30,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742015_1191 (size=14794) 2024-11-26T16:41:30,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:30,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639350509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:30,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639350509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:30,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:30,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:30,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:30,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639350510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:30,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:30,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:30,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639350510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:30,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:30,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639350621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:30,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639350621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:30,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639350621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:30,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639350621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:30,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639350831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:30,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639350831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:30,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639350831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:30,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639350831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:30,899 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:30,903 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267fa07e2a6ab5411ebe5e0a516971c4dd_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267fa07e2a6ab5411ebe5e0a516971c4dd_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:30,904 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/3750b4d3146143008406e00616540e3b, store: [table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:30,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/3750b4d3146143008406e00616540e3b is 175, key is test_row_0/A:col10/1732639290482/Put/seqid=0 2024-11-26T16:41:30,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742016_1192 (size=39749) 2024-11-26T16:41:31,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639351141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:31,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639351141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:31,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:31,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:31,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:31,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639351141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:31,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639351141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:31,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:31,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:31,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:31,310 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/3750b4d3146143008406e00616540e3b 2024-11-26T16:41:31,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/d5cc0cda96e044cc80c370c1d19ad019 is 50, key is test_row_0/B:col10/1732639290482/Put/seqid=0 2024-11-26T16:41:31,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742017_1193 (size=12151) 2024-11-26T16:41:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-26T16:41:31,402 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:31,402 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:41:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-26T16:41:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T16:41:31,404 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:41:31,405 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:41:31,405 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:41:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T16:41:31,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T16:41:31,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:31,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:31,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:31,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:31,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:31,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639351661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:31,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639351661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:31,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639351661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:31,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639351662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:31,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T16:41:31,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:31,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:31,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:31,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:31,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:31,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T16:41:31,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/d5cc0cda96e044cc80c370c1d19ad019 2024-11-26T16:41:31,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/195ef2ae532340f1936e51f9379efc93 is 50, key is test_row_0/C:col10/1732639290482/Put/seqid=0 2024-11-26T16:41:31,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742018_1194 (size=12151) 2024-11-26T16:41:31,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T16:41:31,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:31,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:31,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:31,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:31,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:31,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:32,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T16:41:32,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:32,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:32,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:32,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:32,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:32,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:32,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T16:41:32,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/195ef2ae532340f1936e51f9379efc93 2024-11-26T16:41:32,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/3750b4d3146143008406e00616540e3b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/3750b4d3146143008406e00616540e3b 2024-11-26T16:41:32,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/3750b4d3146143008406e00616540e3b, entries=200, sequenceid=239, filesize=38.8 K 2024-11-26T16:41:32,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/d5cc0cda96e044cc80c370c1d19ad019 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/d5cc0cda96e044cc80c370c1d19ad019 2024-11-26T16:41:32,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T16:41:32,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:32,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:32,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:32,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:32,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:32,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/d5cc0cda96e044cc80c370c1d19ad019, entries=150, sequenceid=239, filesize=11.9 K 2024-11-26T16:41:32,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/195ef2ae532340f1936e51f9379efc93 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/195ef2ae532340f1936e51f9379efc93 2024-11-26T16:41:32,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/195ef2ae532340f1936e51f9379efc93, entries=150, sequenceid=239, filesize=11.9 K 2024-11-26T16:41:32,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 974fac4d1c1c1475caaca19099f1def5 in 1702ms, sequenceid=239, compaction requested=true 2024-11-26T16:41:32,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:32,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:41:32,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:32,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:41:32,187 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:32,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:32,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:41:32,187 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:32,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:32,188 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:32,188 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:32,188 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/B is initiating minor compaction (all files) 2024-11-26T16:41:32,188 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/A is initiating minor compaction (all files) 2024-11-26T16:41:32,188 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/B in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:32,188 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/A in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:32,188 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/94b1cfaf3beb4fb887a6173dac4f7634, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/2e97430fd50f433c8fff60102368a222, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/d5cc0cda96e044cc80c370c1d19ad019] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=36.1 K 2024-11-26T16:41:32,188 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/156d57359b154637be31e97fc6a7d63d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/30a49593e10c495b8c5a92d0513ecf58, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/3750b4d3146143008406e00616540e3b] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=100.0 K 2024-11-26T16:41:32,188 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:32,188 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. files: [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/156d57359b154637be31e97fc6a7d63d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/30a49593e10c495b8c5a92d0513ecf58, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/3750b4d3146143008406e00616540e3b] 2024-11-26T16:41:32,189 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 156d57359b154637be31e97fc6a7d63d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732639286784 2024-11-26T16:41:32,189 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 94b1cfaf3beb4fb887a6173dac4f7634, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732639286784 2024-11-26T16:41:32,189 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e97430fd50f433c8fff60102368a222, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732639287145 2024-11-26T16:41:32,189 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30a49593e10c495b8c5a92d0513ecf58, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732639287145 2024-11-26T16:41:32,190 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting d5cc0cda96e044cc80c370c1d19ad019, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732639289333 2024-11-26T16:41:32,190 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3750b4d3146143008406e00616540e3b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732639289333 2024-11-26T16:41:32,199 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#B#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:32,200 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/77f4e6ddd4fa404cbe9466b68e5ac007 is 50, key is test_row_0/B:col10/1732639290482/Put/seqid=0 2024-11-26T16:41:32,201 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:32,204 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126672133e5a03f45489c360532e650038f_974fac4d1c1c1475caaca19099f1def5 store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:32,207 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126672133e5a03f45489c360532e650038f_974fac4d1c1c1475caaca19099f1def5, store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:32,207 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126672133e5a03f45489c360532e650038f_974fac4d1c1c1475caaca19099f1def5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:32,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742019_1195 (size=12731) 2024-11-26T16:41:32,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742020_1196 (size=4469) 2024-11-26T16:41:32,225 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/77f4e6ddd4fa404cbe9466b68e5ac007 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/77f4e6ddd4fa404cbe9466b68e5ac007 2024-11-26T16:41:32,229 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#A#compaction#163 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:32,229 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/cb19bd13a77d49f3a7fbf8180df5b2bd is 175, key is test_row_0/A:col10/1732639290482/Put/seqid=0 2024-11-26T16:41:32,233 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/B of 974fac4d1c1c1475caaca19099f1def5 into 77f4e6ddd4fa404cbe9466b68e5ac007(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:32,233 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:32,233 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/B, priority=13, startTime=1732639292187; duration=0sec 2024-11-26T16:41:32,233 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:32,234 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:B 2024-11-26T16:41:32,234 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:32,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742021_1197 (size=31685) 2024-11-26T16:41:32,238 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:32,238 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/C is initiating minor compaction (all files) 2024-11-26T16:41:32,238 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/C in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:32,238 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/223284b55a4842b188591499f4c14482, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/a4695b892d924b86a0478335ea554437, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/195ef2ae532340f1936e51f9379efc93] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=36.1 K 2024-11-26T16:41:32,239 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 223284b55a4842b188591499f4c14482, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732639286784 2024-11-26T16:41:32,240 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a4695b892d924b86a0478335ea554437, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732639287145 2024-11-26T16:41:32,241 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 195ef2ae532340f1936e51f9379efc93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732639289333 2024-11-26T16:41:32,244 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/cb19bd13a77d49f3a7fbf8180df5b2bd as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/cb19bd13a77d49f3a7fbf8180df5b2bd 2024-11-26T16:41:32,251 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/A of 974fac4d1c1c1475caaca19099f1def5 into cb19bd13a77d49f3a7fbf8180df5b2bd(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:32,251 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:32,251 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/A, priority=13, startTime=1732639292187; duration=0sec 2024-11-26T16:41:32,252 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:32,252 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:A 2024-11-26T16:41:32,254 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#C#compaction#164 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:32,254 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/84811f0602f04cf9bd8d7c909d281f0b is 50, key is test_row_0/C:col10/1732639290482/Put/seqid=0 2024-11-26T16:41:32,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742022_1198 (size=12731) 2024-11-26T16:41:32,273 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/84811f0602f04cf9bd8d7c909d281f0b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/84811f0602f04cf9bd8d7c909d281f0b 2024-11-26T16:41:32,280 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/C of 974fac4d1c1c1475caaca19099f1def5 into 84811f0602f04cf9bd8d7c909d281f0b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:32,280 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:32,280 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/C, priority=13, startTime=1732639292187; duration=0sec 2024-11-26T16:41:32,280 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:32,280 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:C 2024-11-26T16:41:32,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-26T16:41:32,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:32,327 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 974fac4d1c1c1475caaca19099f1def5 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T16:41:32,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=A 2024-11-26T16:41:32,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:32,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=B 2024-11-26T16:41:32,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:32,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=C 2024-11-26T16:41:32,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:32,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e813c0d7c2bc4cce98649ff6079739ae_974fac4d1c1c1475caaca19099f1def5 is 50, key is test_row_0/A:col10/1732639290508/Put/seqid=0 2024-11-26T16:41:32,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742023_1199 (size=12304) 2024-11-26T16:41:32,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T16:41:32,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:32,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:32,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639352691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:32,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:32,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639352693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:32,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:32,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:32,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:32,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639352695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:32,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:32,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639352697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:32,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:32,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:32,747 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e813c0d7c2bc4cce98649ff6079739ae_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e813c0d7c2bc4cce98649ff6079739ae_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:32,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/96b8bebad48f47338f989bc03aa7b9e4, store: [table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:32,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/96b8bebad48f47338f989bc03aa7b9e4 is 175, key is test_row_0/A:col10/1732639290508/Put/seqid=0 2024-11-26T16:41:32,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742024_1200 (size=31105) 2024-11-26T16:41:32,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639352801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:32,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:32,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639352811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639352811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:32,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:32,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639352811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:32,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:32,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:32,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:33,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639353011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:33,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:33,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639353021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639353021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639353021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:33,162 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/96b8bebad48f47338f989bc03aa7b9e4 2024-11-26T16:41:33,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/a723d1f72a464a1c80c6c7e679478399 is 50, key is test_row_0/B:col10/1732639290508/Put/seqid=0 2024-11-26T16:41:33,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742025_1201 (size=12151) 2024-11-26T16:41:33,190 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/a723d1f72a464a1c80c6c7e679478399 2024-11-26T16:41:33,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/3bbb0b2da6834cc7bd222ecc64fe8362 is 50, key is test_row_0/C:col10/1732639290508/Put/seqid=0 2024-11-26T16:41:33,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742026_1202 (size=12151) 2024-11-26T16:41:33,245 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/3bbb0b2da6834cc7bd222ecc64fe8362 2024-11-26T16:41:33,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/96b8bebad48f47338f989bc03aa7b9e4 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/96b8bebad48f47338f989bc03aa7b9e4 2024-11-26T16:41:33,262 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/96b8bebad48f47338f989bc03aa7b9e4, entries=150, sequenceid=260, filesize=30.4 K 2024-11-26T16:41:33,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/a723d1f72a464a1c80c6c7e679478399 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/a723d1f72a464a1c80c6c7e679478399 2024-11-26T16:41:33,277 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/a723d1f72a464a1c80c6c7e679478399, entries=150, sequenceid=260, filesize=11.9 K 2024-11-26T16:41:33,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/3bbb0b2da6834cc7bd222ecc64fe8362 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/3bbb0b2da6834cc7bd222ecc64fe8362 2024-11-26T16:41:33,291 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/3bbb0b2da6834cc7bd222ecc64fe8362, entries=150, sequenceid=260, filesize=11.9 K 2024-11-26T16:41:33,293 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 974fac4d1c1c1475caaca19099f1def5 in 966ms, sequenceid=260, compaction requested=false 2024-11-26T16:41:33,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:33,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:33,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-26T16:41:33,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-26T16:41:33,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-26T16:41:33,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8920 sec 2024-11-26T16:41:33,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.8990 sec 2024-11-26T16:41:33,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:33,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 974fac4d1c1c1475caaca19099f1def5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T16:41:33,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=A 2024-11-26T16:41:33,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:33,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=B 2024-11-26T16:41:33,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:33,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=C 2024-11-26T16:41:33,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:33,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c401bfb3306748b99ee1c5f9434dab20_974fac4d1c1c1475caaca19099f1def5 is 50, key is test_row_0/A:col10/1732639292693/Put/seqid=0 2024-11-26T16:41:33,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742027_1203 (size=14994) 2024-11-26T16:41:33,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639353364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:33,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639353364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:33,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:33,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:33,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639353368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:33,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:33,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:33,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639353369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:33,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:33,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:33,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639353482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:33,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:33,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639353483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:33,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:33,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:33,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:33,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639353483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:33,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:33,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:33,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639353485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:33,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:33,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-26T16:41:33,552 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:33,553 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:41:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-26T16:41:33,556 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:41:33,557 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:41:33,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:41:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T16:41:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T16:41:33,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639353691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:33,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:33,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639353692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:33,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639353692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:33,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:33,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639353693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:33,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:33,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:33,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T16:41:33,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:33,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:33,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:33,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:33,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:33,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:33,758 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:33,771 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c401bfb3306748b99ee1c5f9434dab20_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c401bfb3306748b99ee1c5f9434dab20_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:33,773 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/1e43d55df4d8419caaa4206cc6cbc20a, store: [table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:33,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/1e43d55df4d8419caaa4206cc6cbc20a is 175, key is test_row_0/A:col10/1732639292693/Put/seqid=0 2024-11-26T16:41:33,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742028_1204 (size=39949) 2024-11-26T16:41:33,794 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/1e43d55df4d8419caaa4206cc6cbc20a 2024-11-26T16:41:33,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/9b22603caf094fb087b6c368e7e90ab1 is 50, key is test_row_0/B:col10/1732639292693/Put/seqid=0 2024-11-26T16:41:33,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T16:41:33,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:33,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:33,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:33,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:33,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:33,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742029_1205 (size=12301) 2024-11-26T16:41:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T16:41:33,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:33,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41648 deadline: 1732639353891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:33,894 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in 974fac4d1c1c1475caaca19099f1def5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:41:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:34,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:34,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639354001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:34,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:34,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639354002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:34,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:34,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:34,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639354002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:34,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:34,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639354002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:34,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:34,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T16:41:34,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:34,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:34,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:34,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:34,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:34,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T16:41:34,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:34,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:34,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:34,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:34,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:34,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:34,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T16:41:34,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/9b22603caf094fb087b6c368e7e90ab1 2024-11-26T16:41:34,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/05ffa44a491a47b59eee5bdc87fc8e10 is 50, key is test_row_0/C:col10/1732639292693/Put/seqid=0 2024-11-26T16:41:34,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742030_1206 (size=12301) 2024-11-26T16:41:34,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/05ffa44a491a47b59eee5bdc87fc8e10 2024-11-26T16:41:34,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T16:41:34,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/1e43d55df4d8419caaa4206cc6cbc20a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/1e43d55df4d8419caaa4206cc6cbc20a 2024-11-26T16:41:34,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:34,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:34,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:34,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:34,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:34,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:34,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/1e43d55df4d8419caaa4206cc6cbc20a, entries=200, sequenceid=275, filesize=39.0 K 2024-11-26T16:41:34,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/9b22603caf094fb087b6c368e7e90ab1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/9b22603caf094fb087b6c368e7e90ab1 2024-11-26T16:41:34,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/9b22603caf094fb087b6c368e7e90ab1, entries=150, sequenceid=275, filesize=12.0 K 2024-11-26T16:41:34,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/05ffa44a491a47b59eee5bdc87fc8e10 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/05ffa44a491a47b59eee5bdc87fc8e10 2024-11-26T16:41:34,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/05ffa44a491a47b59eee5bdc87fc8e10, entries=150, sequenceid=275, filesize=12.0 K 2024-11-26T16:41:34,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 974fac4d1c1c1475caaca19099f1def5 in 1025ms, sequenceid=275, compaction requested=true 2024-11-26T16:41:34,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:34,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:41:34,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:34,354 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:34,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:41:34,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:34,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:41:34,354 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:34,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:34,356 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:34,356 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/B is initiating minor compaction (all files) 2024-11-26T16:41:34,356 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/B in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:34,356 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/77f4e6ddd4fa404cbe9466b68e5ac007, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/a723d1f72a464a1c80c6c7e679478399, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/9b22603caf094fb087b6c368e7e90ab1] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=36.3 K 2024-11-26T16:41:34,356 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:34,356 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/A is initiating minor compaction (all files) 2024-11-26T16:41:34,356 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/A in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:34,357 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/cb19bd13a77d49f3a7fbf8180df5b2bd, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/96b8bebad48f47338f989bc03aa7b9e4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/1e43d55df4d8419caaa4206cc6cbc20a] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=100.3 K 2024-11-26T16:41:34,357 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 77f4e6ddd4fa404cbe9466b68e5ac007, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732639289333 2024-11-26T16:41:34,357 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:34,357 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. files: [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/cb19bd13a77d49f3a7fbf8180df5b2bd, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/96b8bebad48f47338f989bc03aa7b9e4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/1e43d55df4d8419caaa4206cc6cbc20a] 2024-11-26T16:41:34,357 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb19bd13a77d49f3a7fbf8180df5b2bd, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732639289333 2024-11-26T16:41:34,357 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a723d1f72a464a1c80c6c7e679478399, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732639290491 2024-11-26T16:41:34,358 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96b8bebad48f47338f989bc03aa7b9e4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732639290491 2024-11-26T16:41:34,359 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e43d55df4d8419caaa4206cc6cbc20a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732639292691 2024-11-26T16:41:34,359 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b22603caf094fb087b6c368e7e90ab1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732639292693 2024-11-26T16:41:34,369 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:34,371 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#B#compaction#172 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:34,372 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/8ffe6fc309ac4821a4c7b4e5a70a86a1 is 50, key is test_row_0/B:col10/1732639292693/Put/seqid=0 2024-11-26T16:41:34,376 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126d584c1757c9340048748fa7dc19b4049_974fac4d1c1c1475caaca19099f1def5 store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:34,379 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126d584c1757c9340048748fa7dc19b4049_974fac4d1c1c1475caaca19099f1def5, store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:34,379 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d584c1757c9340048748fa7dc19b4049_974fac4d1c1c1475caaca19099f1def5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:34,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742031_1207 (size=12983) 2024-11-26T16:41:34,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742032_1208 (size=4469) 2024-11-26T16:41:34,396 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#A#compaction#171 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:34,397 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/bd346f6f552c431c917e4623fdbae724 is 175, key is test_row_0/A:col10/1732639292693/Put/seqid=0 2024-11-26T16:41:34,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742033_1209 (size=31937) 2024-11-26T16:41:34,422 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/bd346f6f552c431c917e4623fdbae724 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/bd346f6f552c431c917e4623fdbae724 2024-11-26T16:41:34,429 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/A of 974fac4d1c1c1475caaca19099f1def5 into bd346f6f552c431c917e4623fdbae724(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:34,429 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:34,429 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/A, priority=13, startTime=1732639294353; duration=0sec 2024-11-26T16:41:34,429 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:34,429 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:A 2024-11-26T16:41:34,430 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:34,432 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:34,432 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/C is initiating minor compaction (all files) 2024-11-26T16:41:34,432 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/C in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:34,432 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/84811f0602f04cf9bd8d7c909d281f0b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/3bbb0b2da6834cc7bd222ecc64fe8362, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/05ffa44a491a47b59eee5bdc87fc8e10] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=36.3 K 2024-11-26T16:41:34,434 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84811f0602f04cf9bd8d7c909d281f0b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732639289333 2024-11-26T16:41:34,436 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bbb0b2da6834cc7bd222ecc64fe8362, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732639290491 2024-11-26T16:41:34,437 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05ffa44a491a47b59eee5bdc87fc8e10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732639292693 2024-11-26T16:41:34,454 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#C#compaction#173 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:34,455 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/b19b5160221646da8a534309dc590c54 is 50, key is test_row_0/C:col10/1732639292693/Put/seqid=0 2024-11-26T16:41:34,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742034_1210 (size=12983) 2024-11-26T16:41:34,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-26T16:41:34,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:34,486 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 974fac4d1c1c1475caaca19099f1def5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T16:41:34,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=A 2024-11-26T16:41:34,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:34,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=B 2024-11-26T16:41:34,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:34,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=C 2024-11-26T16:41:34,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:34,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f7bf6e60eccf41e6901f66c43aa36437_974fac4d1c1c1475caaca19099f1def5 is 50, key is test_row_0/A:col10/1732639293360/Put/seqid=0 2024-11-26T16:41:34,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742035_1211 (size=12454) 2024-11-26T16:41:34,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:34,512 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f7bf6e60eccf41e6901f66c43aa36437_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f7bf6e60eccf41e6901f66c43aa36437_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:34,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/de6f4eb332a645c891f8da367cf3de9c, store: [table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:34,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/de6f4eb332a645c891f8da367cf3de9c is 175, key is test_row_0/A:col10/1732639293360/Put/seqid=0 2024-11-26T16:41:34,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742036_1212 (size=31255) 2024-11-26T16:41:34,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:34,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:34,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:34,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639354565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:34,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:34,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639354567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:34,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639354567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:34,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:34,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639354569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:34,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:34,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639354672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:34,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:34,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639354681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639354681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:34,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639354693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T16:41:34,792 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/8ffe6fc309ac4821a4c7b4e5a70a86a1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/8ffe6fc309ac4821a4c7b4e5a70a86a1 2024-11-26T16:41:34,798 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/B of 974fac4d1c1c1475caaca19099f1def5 into 8ffe6fc309ac4821a4c7b4e5a70a86a1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:34,798 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:34,798 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/B, priority=13, startTime=1732639294354; duration=0sec 2024-11-26T16:41:34,798 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:34,798 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:B 2024-11-26T16:41:34,880 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/b19b5160221646da8a534309dc590c54 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/b19b5160221646da8a534309dc590c54 2024-11-26T16:41:34,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639354881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:34,889 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/C of 974fac4d1c1c1475caaca19099f1def5 into b19b5160221646da8a534309dc590c54(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:34,889 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:34,889 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/C, priority=13, startTime=1732639294354; duration=0sec 2024-11-26T16:41:34,889 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:34,889 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:C 2024-11-26T16:41:34,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639354892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:34,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:34,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639354892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:34,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:34,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639354911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:34,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:34,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:34,920 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/de6f4eb332a645c891f8da367cf3de9c 2024-11-26T16:41:34,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/399e2d41e6a44b5993ba3b5c4cbfe7e8 is 50, key is test_row_0/B:col10/1732639293360/Put/seqid=0 2024-11-26T16:41:34,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742037_1213 (size=12301) 2024-11-26T16:41:35,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:35,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639355191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:35,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:35,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:35,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:35,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639355201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:35,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:35,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639355211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:35,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:35,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:35,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:35,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639355221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:35,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:35,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:35,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:35,337 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/399e2d41e6a44b5993ba3b5c4cbfe7e8 2024-11-26T16:41:35,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/2c441710b2f94ef2832200a5df51b857 is 50, key is test_row_0/C:col10/1732639293360/Put/seqid=0 2024-11-26T16:41:35,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742038_1214 (size=12301) 2024-11-26T16:41:35,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639355701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:35,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:35,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:35,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T16:41:35,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639355711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:35,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639355731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:35,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:35,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639355731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:35,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:35,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:35,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:35,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:35,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:35,753 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/2c441710b2f94ef2832200a5df51b857 2024-11-26T16:41:35,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/de6f4eb332a645c891f8da367cf3de9c as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/de6f4eb332a645c891f8da367cf3de9c 2024-11-26T16:41:35,764 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/de6f4eb332a645c891f8da367cf3de9c, entries=150, sequenceid=293, filesize=30.5 K 2024-11-26T16:41:35,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/399e2d41e6a44b5993ba3b5c4cbfe7e8 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/399e2d41e6a44b5993ba3b5c4cbfe7e8 2024-11-26T16:41:35,774 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/399e2d41e6a44b5993ba3b5c4cbfe7e8, entries=150, sequenceid=293, filesize=12.0 K 2024-11-26T16:41:35,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/2c441710b2f94ef2832200a5df51b857 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/2c441710b2f94ef2832200a5df51b857 2024-11-26T16:41:35,782 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/2c441710b2f94ef2832200a5df51b857, entries=150, sequenceid=293, filesize=12.0 K 2024-11-26T16:41:35,784 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 974fac4d1c1c1475caaca19099f1def5 in 1297ms, sequenceid=293, compaction requested=false 2024-11-26T16:41:35,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:35,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:35,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-26T16:41:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-26T16:41:35,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-26T16:41:35,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2290 sec 2024-11-26T16:41:35,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.2360 sec 2024-11-26T16:41:36,305 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T16:41:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:36,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 974fac4d1c1c1475caaca19099f1def5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T16:41:36,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=A 2024-11-26T16:41:36,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:36,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=B 2024-11-26T16:41:36,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:36,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=C 2024-11-26T16:41:36,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:36,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f8dcbf8c0416445f87cfcfb40948c750_974fac4d1c1c1475caaca19099f1def5 is 50, key is test_row_0/A:col10/1732639294565/Put/seqid=0 2024-11-26T16:41:36,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742039_1215 (size=14994) 2024-11-26T16:41:36,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639356755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:36,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639356756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:36,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639356758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:36,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:36,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639356759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:36,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:36,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:36,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:36,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:36,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:36,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639356871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:36,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:36,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639356872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:36,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:36,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639356872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:36,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:36,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639356872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:36,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:36,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:36,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:37,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639357082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:37,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639357082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:37,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:37,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639357083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:37,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:37,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639357083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:37,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:37,129 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:37,135 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f8dcbf8c0416445f87cfcfb40948c750_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f8dcbf8c0416445f87cfcfb40948c750_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:37,136 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/2b0313aa22144bea995ce5c971ce7b85, store: [table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:37,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/2b0313aa22144bea995ce5c971ce7b85 is 175, key is test_row_0/A:col10/1732639294565/Put/seqid=0 2024-11-26T16:41:37,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742040_1216 (size=39949) 2024-11-26T16:41:37,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639357392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:37,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639357392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:37,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:37,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639357392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:37,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:37,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:37,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:37,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639357401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:37,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:37,547 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/2b0313aa22144bea995ce5c971ce7b85 2024-11-26T16:41:37,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/ae1db3ee26d14f6092adc1bac7eac6fb is 50, key is test_row_0/B:col10/1732639294565/Put/seqid=0 2024-11-26T16:41:37,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742041_1217 (size=12301) 2024-11-26T16:41:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-26T16:41:37,731 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:37,732 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:41:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-26T16:41:37,735 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:41:37,736 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:41:37,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:41:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T16:41:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T16:41:37,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-26T16:41:37,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:37,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:37,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:37,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:37,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:37,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732639357901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:37,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732639357901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:37,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41660 deadline: 1732639357911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:37,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:37,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732639357911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:37,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:41:37,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/ae1db3ee26d14f6092adc1bac7eac6fb 2024-11-26T16:41:37,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/227e144a20b14c62958dc2a6b8d5004c is 50, key is test_row_0/C:col10/1732639294565/Put/seqid=0 2024-11-26T16:41:38,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742042_1218 (size=12301) 2024-11-26T16:41:38,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/227e144a20b14c62958dc2a6b8d5004c 2024-11-26T16:41:38,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/2b0313aa22144bea995ce5c971ce7b85 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/2b0313aa22144bea995ce5c971ce7b85 2024-11-26T16:41:38,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/2b0313aa22144bea995ce5c971ce7b85, entries=200, sequenceid=312, filesize=39.0 K 2024-11-26T16:41:38,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/ae1db3ee26d14f6092adc1bac7eac6fb as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/ae1db3ee26d14f6092adc1bac7eac6fb 2024-11-26T16:41:38,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/ae1db3ee26d14f6092adc1bac7eac6fb, entries=150, sequenceid=312, filesize=12.0 K 2024-11-26T16:41:38,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/227e144a20b14c62958dc2a6b8d5004c as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/227e144a20b14c62958dc2a6b8d5004c 2024-11-26T16:41:38,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-26T16:41:38,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:38,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. as already flushing 2024-11-26T16:41:38,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:38,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:38,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:38,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/227e144a20b14c62958dc2a6b8d5004c, entries=150, sequenceid=312, filesize=12.0 K 2024-11-26T16:41:38,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 974fac4d1c1c1475caaca19099f1def5 in 1337ms, sequenceid=312, compaction requested=true 2024-11-26T16:41:38,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:38,051 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:38,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:41:38,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:38,051 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T16:41:38,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:41:38,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:38,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 974fac4d1c1c1475caaca19099f1def5:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:41:38,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:38,052 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:38,052 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/A is initiating minor compaction (all files) 2024-11-26T16:41:38,052 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/A in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:38,052 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/bd346f6f552c431c917e4623fdbae724, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/de6f4eb332a645c891f8da367cf3de9c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/2b0313aa22144bea995ce5c971ce7b85] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=100.7 K 2024-11-26T16:41:38,052 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:38,052 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. files: [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/bd346f6f552c431c917e4623fdbae724, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/de6f4eb332a645c891f8da367cf3de9c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/2b0313aa22144bea995ce5c971ce7b85] 2024-11-26T16:41:38,054 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd346f6f552c431c917e4623fdbae724, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732639292693 2024-11-26T16:41:38,054 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting de6f4eb332a645c891f8da367cf3de9c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732639293358 2024-11-26T16:41:38,054 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:38,054 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/B is initiating minor compaction (all files) 2024-11-26T16:41:38,054 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/B in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:38,055 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/8ffe6fc309ac4821a4c7b4e5a70a86a1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/399e2d41e6a44b5993ba3b5c4cbfe7e8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/ae1db3ee26d14f6092adc1bac7eac6fb] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=36.7 K 2024-11-26T16:41:38,055 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b0313aa22144bea995ce5c971ce7b85, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732639294565 2024-11-26T16:41:38,056 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ffe6fc309ac4821a4c7b4e5a70a86a1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732639292693 2024-11-26T16:41:38,057 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 399e2d41e6a44b5993ba3b5c4cbfe7e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732639293358 2024-11-26T16:41:38,057 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting ae1db3ee26d14f6092adc1bac7eac6fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732639294565 2024-11-26T16:41:38,073 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:38,083 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#B#compaction#181 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:38,083 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/21e00fb702334d6087ae6e2592ee02a5 is 50, key is test_row_0/B:col10/1732639294565/Put/seqid=0 2024-11-26T16:41:38,085 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126cc529eb37fe446b6ba8f4c02f7dea36a_974fac4d1c1c1475caaca19099f1def5 store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:38,087 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126cc529eb37fe446b6ba8f4c02f7dea36a_974fac4d1c1c1475caaca19099f1def5, store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:38,087 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126cc529eb37fe446b6ba8f4c02f7dea36a_974fac4d1c1c1475caaca19099f1def5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:38,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742043_1219 (size=13085) 2024-11-26T16:41:38,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742044_1220 (size=4469) 2024-11-26T16:41:38,115 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#A#compaction#180 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:38,116 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/2e8b80281bf743c9853ddecaf9436eab is 175, key is test_row_0/A:col10/1732639294565/Put/seqid=0 2024-11-26T16:41:38,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742045_1221 (size=32039) 2024-11-26T16:41:38,129 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/2e8b80281bf743c9853ddecaf9436eab as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/2e8b80281bf743c9853ddecaf9436eab 2024-11-26T16:41:38,140 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/A of 974fac4d1c1c1475caaca19099f1def5 into 2e8b80281bf743c9853ddecaf9436eab(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:38,140 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:38,140 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/A, priority=13, startTime=1732639298050; duration=0sec 2024-11-26T16:41:38,140 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:38,140 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:A 2024-11-26T16:41:38,141 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:38,144 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:38,144 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 974fac4d1c1c1475caaca19099f1def5/C is initiating minor compaction (all files) 2024-11-26T16:41:38,144 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 974fac4d1c1c1475caaca19099f1def5/C in TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:38,144 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/b19b5160221646da8a534309dc590c54, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/2c441710b2f94ef2832200a5df51b857, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/227e144a20b14c62958dc2a6b8d5004c] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp, totalSize=36.7 K 2024-11-26T16:41:38,145 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting b19b5160221646da8a534309dc590c54, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732639292693 2024-11-26T16:41:38,145 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c441710b2f94ef2832200a5df51b857, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732639293358 2024-11-26T16:41:38,145 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 227e144a20b14c62958dc2a6b8d5004c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732639294565 2024-11-26T16:41:38,158 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 974fac4d1c1c1475caaca19099f1def5#C#compaction#182 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:38,159 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/a84abbf2534f4907acf9401caab02644 is 50, key is test_row_0/C:col10/1732639294565/Put/seqid=0 2024-11-26T16:41:38,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742046_1222 (size=13085) 2024-11-26T16:41:38,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-26T16:41:38,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:38,199 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 974fac4d1c1c1475caaca19099f1def5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T16:41:38,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=A 2024-11-26T16:41:38,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:38,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=B 2024-11-26T16:41:38,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:38,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=C 2024-11-26T16:41:38,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:38,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126eef4ee0f3abc431a9bf6a7d446508784_974fac4d1c1c1475caaca19099f1def5 is 50, key is test_row_0/A:col10/1732639296753/Put/seqid=0 2024-11-26T16:41:38,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742047_1223 (size=12454) 2024-11-26T16:41:38,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:38,251 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126eef4ee0f3abc431a9bf6a7d446508784_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126eef4ee0f3abc431a9bf6a7d446508784_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:38,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/af15e20b2a3c47f0b132e630c7ad40f8, store: [table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:38,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/af15e20b2a3c47f0b132e630c7ad40f8 is 175, key is test_row_0/A:col10/1732639296753/Put/seqid=0 2024-11-26T16:41:38,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742048_1224 (size=31255) 2024-11-26T16:41:38,276 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/af15e20b2a3c47f0b132e630c7ad40f8 2024-11-26T16:41:38,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/5974d43c5bd84ea7b43c9fe126caed8a is 50, key is test_row_0/B:col10/1732639296753/Put/seqid=0 2024-11-26T16:41:38,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742049_1225 (size=12301) 2024-11-26T16:41:38,311 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/5974d43c5bd84ea7b43c9fe126caed8a 2024-11-26T16:41:38,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/40b85d6679724f7394208400561e04a5 is 50, key is test_row_0/C:col10/1732639296753/Put/seqid=0 2024-11-26T16:41:38,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742050_1226 (size=12301) 2024-11-26T16:41:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T16:41:38,509 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/21e00fb702334d6087ae6e2592ee02a5 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/21e00fb702334d6087ae6e2592ee02a5 2024-11-26T16:41:38,517 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-11-26T16:41:38,518 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:41:38,518 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:38,518 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:38,518 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:41:38,522 INFO [Thread-652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-652. 2024-11-26T16:41:38,522 DEBUG [Thread-652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:41:38,522 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:38,522 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:38,522 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:41:38,529 INFO [Thread-654 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-654. 2024-11-26T16:41:38,529 DEBUG [Thread-654 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:41:38,529 DEBUG [Thread-654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:38,529 DEBUG [Thread-654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:38,529 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:41:38,530 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-26T16:41:38,530 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:41:38,530 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:38,530 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:41:38,530 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:38,531 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/B of 974fac4d1c1c1475caaca19099f1def5 into 21e00fb702334d6087ae6e2592ee02a5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:38,531 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:38,531 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/B, priority=13, startTime=1732639298051; duration=0sec 2024-11-26T16:41:38,531 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:38,531 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:B 2024-11-26T16:41:38,577 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/a84abbf2534f4907acf9401caab02644 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/a84abbf2534f4907acf9401caab02644 2024-11-26T16:41:38,582 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 974fac4d1c1c1475caaca19099f1def5/C of 974fac4d1c1c1475caaca19099f1def5 into a84abbf2534f4907acf9401caab02644(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:38,582 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:38,582 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., storeName=974fac4d1c1c1475caaca19099f1def5/C, priority=13, startTime=1732639298052; duration=0sec 2024-11-26T16:41:38,583 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:38,583 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 974fac4d1c1c1475caaca19099f1def5:C 2024-11-26T16:41:38,745 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/40b85d6679724f7394208400561e04a5 2024-11-26T16:41:38,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/af15e20b2a3c47f0b132e630c7ad40f8 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/af15e20b2a3c47f0b132e630c7ad40f8 2024-11-26T16:41:38,755 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/af15e20b2a3c47f0b132e630c7ad40f8, entries=150, sequenceid=328, filesize=30.5 K 2024-11-26T16:41:38,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/5974d43c5bd84ea7b43c9fe126caed8a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/5974d43c5bd84ea7b43c9fe126caed8a 2024-11-26T16:41:38,761 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/5974d43c5bd84ea7b43c9fe126caed8a, entries=150, sequenceid=328, filesize=12.0 K 2024-11-26T16:41:38,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/40b85d6679724f7394208400561e04a5 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/40b85d6679724f7394208400561e04a5 2024-11-26T16:41:38,766 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/40b85d6679724f7394208400561e04a5, entries=150, sequenceid=328, filesize=12.0 K 2024-11-26T16:41:38,767 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 974fac4d1c1c1475caaca19099f1def5 in 568ms, sequenceid=328, compaction requested=false 2024-11-26T16:41:38,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 974fac4d1c1c1475caaca19099f1def5: 2024-11-26T16:41:38,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:38,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-26T16:41:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-26T16:41:38,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-26T16:41:38,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0330 sec 2024-11-26T16:41:38,773 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.0400 sec 2024-11-26T16:41:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-26T16:41:38,871 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:38,913 INFO [Thread-645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-645. 2024-11-26T16:41:38,913 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-26T16:41:38,913 DEBUG [Thread-645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:41:38,913 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:41:38,913 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:38,913 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:38,913 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:38,913 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:38,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:41:38,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:41:38,933 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-26T16:41:38,933 INFO [Thread-641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-641. 2024-11-26T16:41:38,933 DEBUG [Thread-641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:41:38,933 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:41:38,933 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:38,933 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:38,933 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:38,933 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:38,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:41:38,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:41:43,933 INFO [Thread-643 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-643. 2024-11-26T16:41:43,933 DEBUG [Thread-643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:41:43,933 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:43,933 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:43,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:41:43,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T16:41:43,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-26T16:41:43,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-26T16:41:43,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-26T16:41:43,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-26T16:41:43,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-26T16:41:43,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T16:41:43,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2692 2024-11-26T16:41:43,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2831 2024-11-26T16:41:43,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T16:41:43,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1207 2024-11-26T16:41:43,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3617 rows 2024-11-26T16:41:43,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1200 2024-11-26T16:41:43,934 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3598 rows 2024-11-26T16:41:43,934 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T16:41:43,934 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T16:41:43,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:43,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:43,934 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:41:43,935 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T16:41:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T16:41:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T16:41:43,955 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639303955"}]},"ts":"1732639303955"} 2024-11-26T16:41:43,957 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T16:41:43,957 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T16:41:43,957 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T16:41:43,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=974fac4d1c1c1475caaca19099f1def5, UNASSIGN}] 2024-11-26T16:41:43,960 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=974fac4d1c1c1475caaca19099f1def5, UNASSIGN 2024-11-26T16:41:43,961 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=974fac4d1c1c1475caaca19099f1def5, regionState=CLOSING, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:43,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=974fac4d1c1c1475caaca19099f1def5, UNASSIGN because future has completed 2024-11-26T16:41:43,962 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T16:41:43,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134}] 2024-11-26T16:41:44,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T16:41:44,114 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:44,114 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T16:41:44,114 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 974fac4d1c1c1475caaca19099f1def5, disabling compactions & flushes 2024-11-26T16:41:44,114 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:44,115 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:44,115 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. after waiting 0 ms 2024-11-26T16:41:44,115 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:44,115 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 974fac4d1c1c1475caaca19099f1def5 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-26T16:41:44,115 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=A 2024-11-26T16:41:44,115 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:44,115 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=B 2024-11-26T16:41:44,115 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:44,115 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 974fac4d1c1c1475caaca19099f1def5, store=C 2024-11-26T16:41:44,115 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:44,121 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d904f0bbfc614961a44bd0df7a257439_974fac4d1c1c1475caaca19099f1def5 is 50, key is test_row_0/A:col10/1732639298932/Put/seqid=0 2024-11-26T16:41:44,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742051_1227 (size=12454) 2024-11-26T16:41:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T16:41:44,526 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:44,531 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d904f0bbfc614961a44bd0df7a257439_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d904f0bbfc614961a44bd0df7a257439_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:44,531 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/a1ffe1d712564ab5bc011dbf75b9e880, store: [table=TestAcidGuarantees family=A region=974fac4d1c1c1475caaca19099f1def5] 2024-11-26T16:41:44,532 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/a1ffe1d712564ab5bc011dbf75b9e880 is 175, key is test_row_0/A:col10/1732639298932/Put/seqid=0 2024-11-26T16:41:44,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742052_1228 (size=31255) 2024-11-26T16:41:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T16:41:44,936 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/a1ffe1d712564ab5bc011dbf75b9e880 2024-11-26T16:41:44,944 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/ef41d6756749499a995443eaf88df358 is 50, key is test_row_0/B:col10/1732639298932/Put/seqid=0 2024-11-26T16:41:44,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742053_1229 (size=12301) 2024-11-26T16:41:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T16:41:45,348 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/ef41d6756749499a995443eaf88df358 2024-11-26T16:41:45,355 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/f197d97143d4410db9374122cbc66453 is 50, key is test_row_0/C:col10/1732639298932/Put/seqid=0 2024-11-26T16:41:45,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742054_1230 (size=12301) 2024-11-26T16:41:45,759 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/f197d97143d4410db9374122cbc66453 2024-11-26T16:41:45,765 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/A/a1ffe1d712564ab5bc011dbf75b9e880 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/a1ffe1d712564ab5bc011dbf75b9e880 2024-11-26T16:41:45,769 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/a1ffe1d712564ab5bc011dbf75b9e880, entries=150, sequenceid=338, filesize=30.5 K 2024-11-26T16:41:45,769 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/B/ef41d6756749499a995443eaf88df358 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/ef41d6756749499a995443eaf88df358 2024-11-26T16:41:45,773 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/ef41d6756749499a995443eaf88df358, entries=150, sequenceid=338, filesize=12.0 K 2024-11-26T16:41:45,774 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/.tmp/C/f197d97143d4410db9374122cbc66453 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/f197d97143d4410db9374122cbc66453 2024-11-26T16:41:45,778 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/f197d97143d4410db9374122cbc66453, entries=150, sequenceid=338, filesize=12.0 K 2024-11-26T16:41:45,779 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 974fac4d1c1c1475caaca19099f1def5 in 1663ms, sequenceid=338, compaction requested=true 2024-11-26T16:41:45,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/abf2528ef4824b65b1bda57dd128bfae, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/10afa2d184b846d38779851cdabc4ecc, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/3c8762778f06462d8579c67b07506f5b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/cd61b7f25ed44987973d4aca55b20b9b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/cc30350b91c74d1dbf1b32296c42c05f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/4a783db0c5234091bf392ff29e230f23, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/1955a391c41d41e883b9132bd374a770, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/7bcb33f6abd041deb0a3c0934463d70a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/8d2c3ba9aa36458fa48610b541d511c1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/95d508ca20434af2987f4364110f5a71, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/8a8b5954a0d24e7db7c0ff4571d59bab, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/51a610a45df540aa81c4b00fc35f92a2, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/0ef3a3851c9142c6975507058cc4732f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/c8450e1dbecc4be7b4b4d758c3c0b521, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/156d57359b154637be31e97fc6a7d63d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/30a49593e10c495b8c5a92d0513ecf58, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/3750b4d3146143008406e00616540e3b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/cb19bd13a77d49f3a7fbf8180df5b2bd, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/96b8bebad48f47338f989bc03aa7b9e4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/1e43d55df4d8419caaa4206cc6cbc20a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/bd346f6f552c431c917e4623fdbae724, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/de6f4eb332a645c891f8da367cf3de9c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/2b0313aa22144bea995ce5c971ce7b85] to archive 2024-11-26T16:41:45,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T16:41:45,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/abf2528ef4824b65b1bda57dd128bfae to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/abf2528ef4824b65b1bda57dd128bfae 2024-11-26T16:41:45,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/10afa2d184b846d38779851cdabc4ecc to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/10afa2d184b846d38779851cdabc4ecc 2024-11-26T16:41:45,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/3c8762778f06462d8579c67b07506f5b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/3c8762778f06462d8579c67b07506f5b 2024-11-26T16:41:45,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/cd61b7f25ed44987973d4aca55b20b9b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/cd61b7f25ed44987973d4aca55b20b9b 2024-11-26T16:41:45,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/cc30350b91c74d1dbf1b32296c42c05f to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/cc30350b91c74d1dbf1b32296c42c05f 2024-11-26T16:41:45,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/4a783db0c5234091bf392ff29e230f23 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/4a783db0c5234091bf392ff29e230f23 2024-11-26T16:41:45,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/1955a391c41d41e883b9132bd374a770 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/1955a391c41d41e883b9132bd374a770 2024-11-26T16:41:45,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/7bcb33f6abd041deb0a3c0934463d70a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/7bcb33f6abd041deb0a3c0934463d70a 2024-11-26T16:41:45,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/8d2c3ba9aa36458fa48610b541d511c1 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/8d2c3ba9aa36458fa48610b541d511c1 2024-11-26T16:41:45,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/95d508ca20434af2987f4364110f5a71 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/95d508ca20434af2987f4364110f5a71 2024-11-26T16:41:45,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/8a8b5954a0d24e7db7c0ff4571d59bab to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/8a8b5954a0d24e7db7c0ff4571d59bab 2024-11-26T16:41:45,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/51a610a45df540aa81c4b00fc35f92a2 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/51a610a45df540aa81c4b00fc35f92a2 2024-11-26T16:41:45,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/0ef3a3851c9142c6975507058cc4732f to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/0ef3a3851c9142c6975507058cc4732f 2024-11-26T16:41:45,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/c8450e1dbecc4be7b4b4d758c3c0b521 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/c8450e1dbecc4be7b4b4d758c3c0b521 2024-11-26T16:41:45,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/156d57359b154637be31e97fc6a7d63d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/156d57359b154637be31e97fc6a7d63d 2024-11-26T16:41:45,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/30a49593e10c495b8c5a92d0513ecf58 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/30a49593e10c495b8c5a92d0513ecf58 2024-11-26T16:41:45,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/3750b4d3146143008406e00616540e3b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/3750b4d3146143008406e00616540e3b 2024-11-26T16:41:45,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/cb19bd13a77d49f3a7fbf8180df5b2bd to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/cb19bd13a77d49f3a7fbf8180df5b2bd 2024-11-26T16:41:45,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/96b8bebad48f47338f989bc03aa7b9e4 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/96b8bebad48f47338f989bc03aa7b9e4 2024-11-26T16:41:45,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/1e43d55df4d8419caaa4206cc6cbc20a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/1e43d55df4d8419caaa4206cc6cbc20a 2024-11-26T16:41:45,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/bd346f6f552c431c917e4623fdbae724 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/bd346f6f552c431c917e4623fdbae724 2024-11-26T16:41:45,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/de6f4eb332a645c891f8da367cf3de9c to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/de6f4eb332a645c891f8da367cf3de9c 2024-11-26T16:41:45,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/2b0313aa22144bea995ce5c971ce7b85 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/2b0313aa22144bea995ce5c971ce7b85 2024-11-26T16:41:45,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/d534e75cd65b4a789acdb3522adff852, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/96974dc5acbc43a789fe9ebd7f38a58a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/a0709759d30f4cc381537334e699b8d6, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/292dfd79d68e4476a5b42d10df04e25d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/adf93014f9034c668b4095b8636e2060, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/073697526a33413fa9ee7f8d367951a0, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/753426f2e87746f89ad326c4f4a227cc, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/7073103a63184f0b9c7b166a68a2c5b5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/1b5c0401812d427da4de2a27512aab20, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/42331f0fce6646239a157c5f54018e1a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/03eb89e55df0425fae9111794b8efc3e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/d9feef9f0bf348a09d3d90d6424b02fa, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/f9da74b279ce4ca7a0b13e4daf91865c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/94b1cfaf3beb4fb887a6173dac4f7634, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/f16fca1f19ba459797bab065b7df1609, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/2e97430fd50f433c8fff60102368a222, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/77f4e6ddd4fa404cbe9466b68e5ac007, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/d5cc0cda96e044cc80c370c1d19ad019, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/a723d1f72a464a1c80c6c7e679478399, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/8ffe6fc309ac4821a4c7b4e5a70a86a1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/9b22603caf094fb087b6c368e7e90ab1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/399e2d41e6a44b5993ba3b5c4cbfe7e8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/ae1db3ee26d14f6092adc1bac7eac6fb] to archive 2024-11-26T16:41:45,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T16:41:45,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/d534e75cd65b4a789acdb3522adff852 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/d534e75cd65b4a789acdb3522adff852 2024-11-26T16:41:45,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/96974dc5acbc43a789fe9ebd7f38a58a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/96974dc5acbc43a789fe9ebd7f38a58a 2024-11-26T16:41:45,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/a0709759d30f4cc381537334e699b8d6 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/a0709759d30f4cc381537334e699b8d6 2024-11-26T16:41:45,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/292dfd79d68e4476a5b42d10df04e25d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/292dfd79d68e4476a5b42d10df04e25d 2024-11-26T16:41:45,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/adf93014f9034c668b4095b8636e2060 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/adf93014f9034c668b4095b8636e2060 2024-11-26T16:41:45,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/073697526a33413fa9ee7f8d367951a0 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/073697526a33413fa9ee7f8d367951a0 2024-11-26T16:41:45,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/753426f2e87746f89ad326c4f4a227cc to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/753426f2e87746f89ad326c4f4a227cc 2024-11-26T16:41:45,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/7073103a63184f0b9c7b166a68a2c5b5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/7073103a63184f0b9c7b166a68a2c5b5 2024-11-26T16:41:45,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/1b5c0401812d427da4de2a27512aab20 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/1b5c0401812d427da4de2a27512aab20 2024-11-26T16:41:45,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/42331f0fce6646239a157c5f54018e1a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/42331f0fce6646239a157c5f54018e1a 2024-11-26T16:41:45,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/03eb89e55df0425fae9111794b8efc3e to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/03eb89e55df0425fae9111794b8efc3e 2024-11-26T16:41:45,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/d9feef9f0bf348a09d3d90d6424b02fa to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/d9feef9f0bf348a09d3d90d6424b02fa 2024-11-26T16:41:45,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/f9da74b279ce4ca7a0b13e4daf91865c to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/f9da74b279ce4ca7a0b13e4daf91865c 2024-11-26T16:41:45,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/94b1cfaf3beb4fb887a6173dac4f7634 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/94b1cfaf3beb4fb887a6173dac4f7634 2024-11-26T16:41:45,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/f16fca1f19ba459797bab065b7df1609 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/f16fca1f19ba459797bab065b7df1609 2024-11-26T16:41:45,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/2e97430fd50f433c8fff60102368a222 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/2e97430fd50f433c8fff60102368a222 2024-11-26T16:41:45,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/77f4e6ddd4fa404cbe9466b68e5ac007 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/77f4e6ddd4fa404cbe9466b68e5ac007 2024-11-26T16:41:45,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/d5cc0cda96e044cc80c370c1d19ad019 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/d5cc0cda96e044cc80c370c1d19ad019 2024-11-26T16:41:45,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/a723d1f72a464a1c80c6c7e679478399 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/a723d1f72a464a1c80c6c7e679478399 2024-11-26T16:41:45,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/8ffe6fc309ac4821a4c7b4e5a70a86a1 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/8ffe6fc309ac4821a4c7b4e5a70a86a1 2024-11-26T16:41:45,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/9b22603caf094fb087b6c368e7e90ab1 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/9b22603caf094fb087b6c368e7e90ab1 2024-11-26T16:41:45,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/399e2d41e6a44b5993ba3b5c4cbfe7e8 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/399e2d41e6a44b5993ba3b5c4cbfe7e8 2024-11-26T16:41:45,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/ae1db3ee26d14f6092adc1bac7eac6fb to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/ae1db3ee26d14f6092adc1bac7eac6fb 2024-11-26T16:41:45,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/c8358d829fab49c4bdf4487ea64943f9, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/291d03cc66c348a18995f4d83f266aa8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/944f5ffd08ff4ebcaefd1b80f3e81273, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/8a55fb8560754e5eb665fb6e054b9942, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/80b5e407c2ba4006844ce09dcb57284b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/01a870d7328b420da79fbc4dfd12292f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/3915008a8ac342cbb9b61b6fd3cd0662, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/68c9a2395ebb4be8a94f52390df0bd00, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/a307b24633a34f7f8993f90aee05ac25, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/149846a986a2431ba298c966e2311cbd, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/695ba50dcc404791910630688d018f2c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/90851ecf66c9487eb81d8d08bcd52fe4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/3f4aff44ab54421c8d8d4b0632446861, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/223284b55a4842b188591499f4c14482, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/04dabd59eb214d698fed0626cd015286, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/a4695b892d924b86a0478335ea554437, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/84811f0602f04cf9bd8d7c909d281f0b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/195ef2ae532340f1936e51f9379efc93, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/3bbb0b2da6834cc7bd222ecc64fe8362, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/b19b5160221646da8a534309dc590c54, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/05ffa44a491a47b59eee5bdc87fc8e10, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/2c441710b2f94ef2832200a5df51b857, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/227e144a20b14c62958dc2a6b8d5004c] to archive 2024-11-26T16:41:45,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T16:41:45,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/c8358d829fab49c4bdf4487ea64943f9 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/c8358d829fab49c4bdf4487ea64943f9 2024-11-26T16:41:45,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/291d03cc66c348a18995f4d83f266aa8 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/291d03cc66c348a18995f4d83f266aa8 2024-11-26T16:41:45,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/944f5ffd08ff4ebcaefd1b80f3e81273 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/944f5ffd08ff4ebcaefd1b80f3e81273 2024-11-26T16:41:45,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/8a55fb8560754e5eb665fb6e054b9942 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/8a55fb8560754e5eb665fb6e054b9942 2024-11-26T16:41:45,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/80b5e407c2ba4006844ce09dcb57284b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/80b5e407c2ba4006844ce09dcb57284b 2024-11-26T16:41:45,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/01a870d7328b420da79fbc4dfd12292f to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/01a870d7328b420da79fbc4dfd12292f 2024-11-26T16:41:45,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/3915008a8ac342cbb9b61b6fd3cd0662 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/3915008a8ac342cbb9b61b6fd3cd0662 2024-11-26T16:41:45,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/68c9a2395ebb4be8a94f52390df0bd00 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/68c9a2395ebb4be8a94f52390df0bd00 2024-11-26T16:41:45,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/a307b24633a34f7f8993f90aee05ac25 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/a307b24633a34f7f8993f90aee05ac25 2024-11-26T16:41:45,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/149846a986a2431ba298c966e2311cbd to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/149846a986a2431ba298c966e2311cbd 2024-11-26T16:41:45,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/695ba50dcc404791910630688d018f2c to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/695ba50dcc404791910630688d018f2c 2024-11-26T16:41:45,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/90851ecf66c9487eb81d8d08bcd52fe4 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/90851ecf66c9487eb81d8d08bcd52fe4 2024-11-26T16:41:45,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/3f4aff44ab54421c8d8d4b0632446861 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/3f4aff44ab54421c8d8d4b0632446861 2024-11-26T16:41:45,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/223284b55a4842b188591499f4c14482 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/223284b55a4842b188591499f4c14482 2024-11-26T16:41:45,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/04dabd59eb214d698fed0626cd015286 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/04dabd59eb214d698fed0626cd015286 2024-11-26T16:41:45,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/a4695b892d924b86a0478335ea554437 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/a4695b892d924b86a0478335ea554437 2024-11-26T16:41:45,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/84811f0602f04cf9bd8d7c909d281f0b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/84811f0602f04cf9bd8d7c909d281f0b 2024-11-26T16:41:45,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/195ef2ae532340f1936e51f9379efc93 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/195ef2ae532340f1936e51f9379efc93 2024-11-26T16:41:45,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/3bbb0b2da6834cc7bd222ecc64fe8362 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/3bbb0b2da6834cc7bd222ecc64fe8362 2024-11-26T16:41:45,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/b19b5160221646da8a534309dc590c54 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/b19b5160221646da8a534309dc590c54 2024-11-26T16:41:45,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/05ffa44a491a47b59eee5bdc87fc8e10 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/05ffa44a491a47b59eee5bdc87fc8e10 2024-11-26T16:41:45,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/2c441710b2f94ef2832200a5df51b857 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/2c441710b2f94ef2832200a5df51b857 2024-11-26T16:41:45,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/227e144a20b14c62958dc2a6b8d5004c to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/227e144a20b14c62958dc2a6b8d5004c 2024-11-26T16:41:45,865 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/recovered.edits/341.seqid, newMaxSeqId=341, maxSeqId=4 2024-11-26T16:41:45,866 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. 2024-11-26T16:41:45,866 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 974fac4d1c1c1475caaca19099f1def5: Waiting for close lock at 1732639304114Running coprocessor pre-close hooks at 1732639304114Disabling compacts and flushes for region at 1732639304114Disabling writes for close at 1732639304115 (+1 ms)Obtaining lock to block concurrent updates at 1732639304115Preparing flush snapshotting stores in 974fac4d1c1c1475caaca19099f1def5 at 1732639304115Finished memstore snapshotting TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732639304115Flushing stores of TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5. at 1732639304116 (+1 ms)Flushing 974fac4d1c1c1475caaca19099f1def5/A: creating writer at 1732639304116Flushing mob file 974fac4d1c1c1475caaca19099f1def5/A: appending metadata at 1732639304121 (+5 ms)Flushing mob file 974fac4d1c1c1475caaca19099f1def5/A: closing flushed file at 1732639304121Flushing 974fac4d1c1c1475caaca19099f1def5/A: appending metadata at 1732639304531 (+410 ms)Flushing 974fac4d1c1c1475caaca19099f1def5/A: closing flushed file at 1732639304532 (+1 ms)Flushing 974fac4d1c1c1475caaca19099f1def5/B: creating writer at 1732639304941 (+409 ms)Flushing 974fac4d1c1c1475caaca19099f1def5/B: appending metadata at 1732639304943 (+2 ms)Flushing 974fac4d1c1c1475caaca19099f1def5/B: closing flushed file at 1732639304943Flushing 974fac4d1c1c1475caaca19099f1def5/C: creating writer at 1732639305352 (+409 ms)Flushing 974fac4d1c1c1475caaca19099f1def5/C: appending metadata at 1732639305355 (+3 ms)Flushing 974fac4d1c1c1475caaca19099f1def5/C: closing flushed file at 1732639305355Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@471bb84c: reopening flushed file at 1732639305764 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ab961fb: reopening flushed file at 1732639305769 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7686365d: reopening flushed file at 1732639305773 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 974fac4d1c1c1475caaca19099f1def5 in 1663ms, sequenceid=338, compaction requested=true at 1732639305779 (+6 ms)Writing region close event to WAL at 1732639305862 (+83 ms)Running coprocessor post-close hooks at 1732639305866 (+4 ms)Closed at 1732639305866 2024-11-26T16:41:45,868 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:45,868 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=974fac4d1c1c1475caaca19099f1def5, regionState=CLOSED 2024-11-26T16:41:45,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 because future has completed 2024-11-26T16:41:45,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-26T16:41:45,873 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 974fac4d1c1c1475caaca19099f1def5, server=2dedfe7466e8,38109,1732639238134 in 1.9080 sec 2024-11-26T16:41:45,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-26T16:41:45,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=974fac4d1c1c1475caaca19099f1def5, UNASSIGN in 1.9130 sec 2024-11-26T16:41:45,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-26T16:41:45,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9170 sec 2024-11-26T16:41:45,877 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639305877"}]},"ts":"1732639305877"} 2024-11-26T16:41:45,878 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T16:41:45,878 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T16:41:45,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9430 sec 2024-11-26T16:41:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-26T16:41:46,102 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:46,102 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T16:41:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:41:46,104 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:41:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T16:41:46,105 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:41:46,107 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,109 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A, FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B, FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C, FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/recovered.edits] 2024-11-26T16:41:46,112 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/2e8b80281bf743c9853ddecaf9436eab to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/2e8b80281bf743c9853ddecaf9436eab 2024-11-26T16:41:46,113 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/a1ffe1d712564ab5bc011dbf75b9e880 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/a1ffe1d712564ab5bc011dbf75b9e880 2024-11-26T16:41:46,114 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/af15e20b2a3c47f0b132e630c7ad40f8 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/A/af15e20b2a3c47f0b132e630c7ad40f8 2024-11-26T16:41:46,116 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/21e00fb702334d6087ae6e2592ee02a5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/21e00fb702334d6087ae6e2592ee02a5 2024-11-26T16:41:46,117 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/5974d43c5bd84ea7b43c9fe126caed8a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/5974d43c5bd84ea7b43c9fe126caed8a 2024-11-26T16:41:46,118 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/ef41d6756749499a995443eaf88df358 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/B/ef41d6756749499a995443eaf88df358 2024-11-26T16:41:46,120 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/40b85d6679724f7394208400561e04a5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/40b85d6679724f7394208400561e04a5 2024-11-26T16:41:46,121 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/a84abbf2534f4907acf9401caab02644 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/a84abbf2534f4907acf9401caab02644 2024-11-26T16:41:46,122 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/f197d97143d4410db9374122cbc66453 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/C/f197d97143d4410db9374122cbc66453 2024-11-26T16:41:46,125 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/recovered.edits/341.seqid to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5/recovered.edits/341.seqid 2024-11-26T16:41:46,126 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,126 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T16:41:46,126 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T16:41:46,127 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T16:41:46,131 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126023360b3e4aa46e4bf0d6ad2e1947f59_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126023360b3e4aa46e4bf0d6ad2e1947f59_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,132 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126094043e5b5de4e5cb480849c19ea7512_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126094043e5b5de4e5cb480849c19ea7512_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,133 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112616880becc0024f449e05d7329ed8bc2c_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112616880becc0024f449e05d7329ed8bc2c_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,135 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126249e34dce5d94f899ae2824da744f09e_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126249e34dce5d94f899ae2824da744f09e_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,136 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262cf5c0c448ce4e7ab65e3a66d08b16e4_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262cf5c0c448ce4e7ab65e3a66d08b16e4_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,137 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126438db52ab3a24095bdf2085278406d51_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126438db52ab3a24095bdf2085278406d51_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,138 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112675a2e82da4134e008a5289937874e6c6_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112675a2e82da4134e008a5289937874e6c6_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,139 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267c0e7f8bef5f47018a133dc6b497adc0_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267c0e7f8bef5f47018a133dc6b497adc0_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,140 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267fa07e2a6ab5411ebe5e0a516971c4dd_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267fa07e2a6ab5411ebe5e0a516971c4dd_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,141 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411268f9257e48c4a4ef2a1a68913128901bc_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411268f9257e48c4a4ef2a1a68913128901bc_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,142 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b242bd13d8374d08926bbfa88d9c5cf1_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b242bd13d8374d08926bbfa88d9c5cf1_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,143 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c401bfb3306748b99ee1c5f9434dab20_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126c401bfb3306748b99ee1c5f9434dab20_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,144 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cfa882ee9ef44177be2148ecd831a8b3_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cfa882ee9ef44177be2148ecd831a8b3_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,145 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d904f0bbfc614961a44bd0df7a257439_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d904f0bbfc614961a44bd0df7a257439_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,146 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e813c0d7c2bc4cce98649ff6079739ae_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e813c0d7c2bc4cce98649ff6079739ae_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,147 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126eef4ee0f3abc431a9bf6a7d446508784_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126eef4ee0f3abc431a9bf6a7d446508784_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,148 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f243f76cbb3a41b2beff9efdc6a89cf4_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f243f76cbb3a41b2beff9efdc6a89cf4_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,149 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f7bf6e60eccf41e6901f66c43aa36437_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f7bf6e60eccf41e6901f66c43aa36437_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,150 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f8dcbf8c0416445f87cfcfb40948c750_974fac4d1c1c1475caaca19099f1def5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f8dcbf8c0416445f87cfcfb40948c750_974fac4d1c1c1475caaca19099f1def5 2024-11-26T16:41:46,151 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T16:41:46,153 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:41:46,156 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T16:41:46,158 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T16:41:46,159 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:41:46,159 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T16:41:46,159 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732639306159"}]},"ts":"9223372036854775807"} 2024-11-26T16:41:46,161 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T16:41:46,161 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 974fac4d1c1c1475caaca19099f1def5, NAME => 'TestAcidGuarantees,,1732639275306.974fac4d1c1c1475caaca19099f1def5.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T16:41:46,161 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T16:41:46,161 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732639306161"}]},"ts":"9223372036854775807"} 2024-11-26T16:41:46,163 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T16:41:46,164 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:41:46,165 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-11-26T16:41:46,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-26T16:41:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T16:41:46,212 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:46,221 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 247) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/cluster_5c35ce1e-289c-85d9-824c-12c667b295d2/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2059946220_22 at /127.0.0.1:55688 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/cluster_5c35ce1e-289c-85d9-824c-12c667b295d2/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=447 (was 390) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5949 (was 6537) 2024-11-26T16:41:46,230 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=447, ProcessCount=11, AvailableMemoryMB=5949 2024-11-26T16:41:46,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T16:41:46,231 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T16:41:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T16:41:46,233 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T16:41:46,234 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:46,234 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-26T16:41:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-26T16:41:46,234 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T16:41:46,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742055_1231 (size=1061) 2024-11-26T16:41:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-26T16:41:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-26T16:41:46,642 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711 2024-11-26T16:41:46,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742056_1232 (size=53) 2024-11-26T16:41:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-26T16:41:47,048 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T16:41:47,048 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8c8b9347f011e2219af1b66f205e204f, disabling compactions & flushes 2024-11-26T16:41:47,049 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:47,049 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:47,049 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. after waiting 0 ms 2024-11-26T16:41:47,049 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:47,049 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:47,049 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8c8b9347f011e2219af1b66f205e204f: Waiting for close lock at 1732639307048Disabling compacts and flushes for region at 1732639307048Disabling writes for close at 1732639307049 (+1 ms)Writing region close event to WAL at 1732639307049Closed at 1732639307049 2024-11-26T16:41:47,050 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T16:41:47,050 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732639307050"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732639307050"}]},"ts":"1732639307050"} 2024-11-26T16:41:47,051 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T16:41:47,052 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T16:41:47,052 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639307052"}]},"ts":"1732639307052"} 2024-11-26T16:41:47,053 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T16:41:47,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8c8b9347f011e2219af1b66f205e204f, ASSIGN}] 2024-11-26T16:41:47,055 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8c8b9347f011e2219af1b66f205e204f, ASSIGN 2024-11-26T16:41:47,055 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8c8b9347f011e2219af1b66f205e204f, ASSIGN; state=OFFLINE, location=2dedfe7466e8,38109,1732639238134; forceNewPlan=false, retain=false 2024-11-26T16:41:47,206 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=8c8b9347f011e2219af1b66f205e204f, regionState=OPENING, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:47,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8c8b9347f011e2219af1b66f205e204f, ASSIGN because future has completed 2024-11-26T16:41:47,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134}] 2024-11-26T16:41:47,362 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:47,363 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} 2024-11-26T16:41:47,363 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:47,363 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T16:41:47,363 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:47,363 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:47,364 INFO [StoreOpener-8c8b9347f011e2219af1b66f205e204f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:47,366 INFO [StoreOpener-8c8b9347f011e2219af1b66f205e204f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:41:47,366 INFO [StoreOpener-8c8b9347f011e2219af1b66f205e204f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8c8b9347f011e2219af1b66f205e204f columnFamilyName A 2024-11-26T16:41:47,366 DEBUG [StoreOpener-8c8b9347f011e2219af1b66f205e204f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:47,366 INFO [StoreOpener-8c8b9347f011e2219af1b66f205e204f-1 {}] regionserver.HStore(327): Store=8c8b9347f011e2219af1b66f205e204f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:41:47,366 INFO [StoreOpener-8c8b9347f011e2219af1b66f205e204f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:47,367 INFO [StoreOpener-8c8b9347f011e2219af1b66f205e204f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:41:47,367 INFO [StoreOpener-8c8b9347f011e2219af1b66f205e204f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8c8b9347f011e2219af1b66f205e204f columnFamilyName B 2024-11-26T16:41:47,367 DEBUG [StoreOpener-8c8b9347f011e2219af1b66f205e204f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:47,368 INFO [StoreOpener-8c8b9347f011e2219af1b66f205e204f-1 {}] regionserver.HStore(327): Store=8c8b9347f011e2219af1b66f205e204f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:41:47,368 INFO [StoreOpener-8c8b9347f011e2219af1b66f205e204f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:47,369 INFO [StoreOpener-8c8b9347f011e2219af1b66f205e204f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:41:47,369 INFO [StoreOpener-8c8b9347f011e2219af1b66f205e204f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8c8b9347f011e2219af1b66f205e204f columnFamilyName C 2024-11-26T16:41:47,369 DEBUG [StoreOpener-8c8b9347f011e2219af1b66f205e204f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:41:47,369 INFO [StoreOpener-8c8b9347f011e2219af1b66f205e204f-1 {}] regionserver.HStore(327): Store=8c8b9347f011e2219af1b66f205e204f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:41:47,369 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:47,370 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:47,370 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:47,370 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-26T16:41:47,371 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:47,371 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:47,372 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T16:41:47,373 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:47,374 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T16:41:47,375 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 8c8b9347f011e2219af1b66f205e204f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60492894, jitterRate=-0.09858563542366028}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T16:41:47,375 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:47,375 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 8c8b9347f011e2219af1b66f205e204f: Running coprocessor pre-open hook at 1732639307363Writing region info on filesystem at 1732639307363Initializing all the Stores at 1732639307364 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639307364Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639307364Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639307364Cleaning up temporary data from old regions at 1732639307371 (+7 ms)Running coprocessor post-open hooks at 1732639307375 (+4 ms)Region opened successfully at 1732639307375 2024-11-26T16:41:47,376 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., pid=63, masterSystemTime=1732639307359 2024-11-26T16:41:47,378 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:47,378 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:47,378 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=8c8b9347f011e2219af1b66f205e204f, regionState=OPEN, openSeqNum=2, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:47,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 because future has completed 2024-11-26T16:41:47,382 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-26T16:41:47,382 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 in 172 msec 2024-11-26T16:41:47,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-26T16:41:47,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8c8b9347f011e2219af1b66f205e204f, ASSIGN in 328 msec 2024-11-26T16:41:47,384 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T16:41:47,384 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639307384"}]},"ts":"1732639307384"} 2024-11-26T16:41:47,385 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T16:41:47,386 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T16:41:47,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1540 sec 2024-11-26T16:41:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-26T16:41:48,382 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:48,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c45ccf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:48,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:48,384 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dd0b8f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:48,385 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@734fd3d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,386 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:48,388 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:48,389 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T16:41:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 2dedfe7466e8,35233,1732639237322 2024-11-26T16:41:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4839bbe7 2024-11-26T16:41:48,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T16:41:48,391 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T16:41:48,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e140226, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,392 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:48,392 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:48,392 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:48,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25990f56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:48,394 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:48,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15700b49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@279bf308, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,395 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:48,395 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:48,395 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:48,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:48,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:48,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ee7c404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:48,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:48,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:48,397 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:48,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a18fd98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,398 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ba0760, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,398 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:48,398 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:48,398 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52626491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:48,400 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:48,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b68bb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@102b3c4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,401 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:48,401 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:48,401 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a8f1bd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:48,402 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:48,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6550973b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@574b1ec0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,403 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:48,403 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:48,404 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337f2e86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:48,405 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:48,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@385f1c9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b646580, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,406 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:48,406 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:48,407 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:48,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:48,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:48,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6365df14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:48,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:48,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:48,408 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54d327d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c85ada2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,409 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:48,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:48,410 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:48,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:48,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:48,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19ab3c3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:48,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:48,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:48,411 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:48,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f920570, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3660e239, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,412 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:48,412 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:48,413 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f876bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:48,414 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:48,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@396c5f30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67353b2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,415 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:48,415 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:48,416 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a679399, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:48,417 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57498, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:48,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e19aaf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@364bb93f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,418 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:41:48,418 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:41:48,418 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:41:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:41:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:41:48,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d87d72e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:41:48,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:41:48,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:41:48,419 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:41:48,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3664a32c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:41:48,421 DEBUG [Thread-1091 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:48,421 DEBUG [Thread-1098 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:48,421 DEBUG [Thread-1087 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:48,421 DEBUG [Thread-1089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:48,421 DEBUG [Thread-1106 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:48,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:41:48,421 DEBUG [Thread-1100 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:48,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:48,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:48,421 DEBUG [Thread-1104 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:48,422 DEBUG [Thread-1095 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:48,422 DEBUG [Thread-1093 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:48,423 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:48,423 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:48,423 DEBUG [Thread-1102 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:41:48,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-26T16:41:48,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:48,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:48,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:48,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:48,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:48,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:48,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:48,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:48,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:48,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:41:48,426 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:41:48,426 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:48,426 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:41:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:41:48,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:41:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T16:41:48,427 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:48,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:48,428 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:41:48,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:41:48,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:48,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:41:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:41:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:41:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:41:48,431 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:41:48,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:41:48,431 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:41:48,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:41:48,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:41:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:48,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T16:41:48,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:41:48,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:48,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:41:48,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:48,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:41:48,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:48,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/0b795fd6e586474a8a955eb519e756b9 is 50, key is test_row_0/A:col10/1732639308432/Put/seqid=0 2024-11-26T16:41:48,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639368467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:48,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732639308463, max timestamp=1732639308471Num uniques 859; 2024-11-26T16:41:48,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732639308463, max timestamp=1732639308471Num uniques 859; 2024-11-26T16:41:48,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:48,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732639308463, max timestamp=1732639308471Num uniques 859; 2024-11-26T16:41:48,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T16:41:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639368468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:48,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T16:41:48,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:48,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T16:41:48,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:48,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742057_1233 (size=12001) 2024-11-26T16:41:48,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:48,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639368509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639368510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:48,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:48,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639368511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:48,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:48,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T16:41:48,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:48,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639368581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:48,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T16:41:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:48,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:48,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:48,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:48,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:48,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:48,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639368581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:48,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639368621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:48,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639368621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:48,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639368621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:48,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T16:41:48,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:48,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:48,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:48,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:48,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:48,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T16:41:48,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639368791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:48,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639368791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:48,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:48,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639368831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:48,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:48,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639368831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:48,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:48,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:48,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639368831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:48,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:48,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:48,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:48,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/0b795fd6e586474a8a955eb519e756b9 2024-11-26T16:41:48,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T16:41:48,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:48,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:48,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:48,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:48,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:48,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:48,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/a7c3d7e7722f4dbebb4a7198033ad0bc is 50, key is test_row_0/B:col10/1732639308432/Put/seqid=0 2024-11-26T16:41:48,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742058_1234 (size=12001) 2024-11-26T16:41:48,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/a7c3d7e7722f4dbebb4a7198033ad0bc 2024-11-26T16:41:48,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/a22304964d794101b872cbc50a16d386 is 50, key is test_row_0/C:col10/1732639308432/Put/seqid=0 2024-11-26T16:41:48,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742059_1235 (size=12001) 2024-11-26T16:41:49,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T16:41:49,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:49,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:49,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:49,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:49,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:49,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T16:41:49,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639369101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:49,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639369101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:49,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:49,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:49,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639369140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:49,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639369141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:49,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:49,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639369141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:49,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T16:41:49,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:49,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:49,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:49,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:49,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:49,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:49,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/a22304964d794101b872cbc50a16d386 2024-11-26T16:41:49,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T16:41:49,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:49,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:49,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:49,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:49,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:49,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:49,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/0b795fd6e586474a8a955eb519e756b9 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/0b795fd6e586474a8a955eb519e756b9 2024-11-26T16:41:49,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/0b795fd6e586474a8a955eb519e756b9, entries=150, sequenceid=12, filesize=11.7 K 2024-11-26T16:41:49,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/a7c3d7e7722f4dbebb4a7198033ad0bc as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a7c3d7e7722f4dbebb4a7198033ad0bc 2024-11-26T16:41:49,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a7c3d7e7722f4dbebb4a7198033ad0bc, entries=150, sequenceid=12, filesize=11.7 K 2024-11-26T16:41:49,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/a22304964d794101b872cbc50a16d386 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/a22304964d794101b872cbc50a16d386 2024-11-26T16:41:49,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/a22304964d794101b872cbc50a16d386, entries=150, sequenceid=12, filesize=11.7 K 2024-11-26T16:41:49,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8c8b9347f011e2219af1b66f205e204f in 936ms, sequenceid=12, compaction requested=false 2024-11-26T16:41:49,372 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T16:41:49,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:49,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-26T16:41:49,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:49,506 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-26T16:41:49,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:41:49,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:41:49,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:41:49,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:41:49,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:41:49,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:41:49,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/a9f75cc33df94d0d91e8a0519a0f173b is 50, key is test_row_0/A:col10/1732639308508/Put/seqid=0 2024-11-26T16:41:49,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742060_1236 (size=12001) 2024-11-26T16:41:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T16:41:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:49,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:49,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639369631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:49,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:49,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639369633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:49,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:49,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639369651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:49,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639369651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:49,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:49,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:49,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639369651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:49,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:49,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:49,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639369741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:49,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:49,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639369741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:49,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:49,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:49,917 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/a9f75cc33df94d0d91e8a0519a0f173b 2024-11-26T16:41:49,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/86f6f86ce2d849da9cdc44ecb0094b7d is 50, key is test_row_0/B:col10/1732639308508/Put/seqid=0 2024-11-26T16:41:49,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742061_1237 (size=12001) 2024-11-26T16:41:49,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639369951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:49,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639369951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:50,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:50,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639370261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:50,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:50,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639370262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:50,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:50,335 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/86f6f86ce2d849da9cdc44ecb0094b7d 2024-11-26T16:41:50,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/3cf658b406fd43fbbed930d90560ca82 is 50, key is test_row_0/C:col10/1732639308508/Put/seqid=0 2024-11-26T16:41:50,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742062_1238 (size=12001) 2024-11-26T16:41:50,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T16:41:50,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639370661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:50,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639370661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:50,663 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in 8c8b9347f011e2219af1b66f205e204f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:41:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:50,663 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in 8c8b9347f011e2219af1b66f205e204f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:41:50,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:50,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:50,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:50,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:50,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639370671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:50,674 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in 8c8b9347f011e2219af1b66f205e204f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:41:50,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:50,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:50,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:50,750 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/3cf658b406fd43fbbed930d90560ca82 2024-11-26T16:41:50,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/a9f75cc33df94d0d91e8a0519a0f173b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/a9f75cc33df94d0d91e8a0519a0f173b 2024-11-26T16:41:50,759 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/a9f75cc33df94d0d91e8a0519a0f173b, entries=150, sequenceid=46, filesize=11.7 K 2024-11-26T16:41:50,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/86f6f86ce2d849da9cdc44ecb0094b7d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/86f6f86ce2d849da9cdc44ecb0094b7d 2024-11-26T16:41:50,764 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/86f6f86ce2d849da9cdc44ecb0094b7d, entries=150, sequenceid=46, filesize=11.7 K 2024-11-26T16:41:50,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/3cf658b406fd43fbbed930d90560ca82 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/3cf658b406fd43fbbed930d90560ca82 2024-11-26T16:41:50,769 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/3cf658b406fd43fbbed930d90560ca82, entries=150, sequenceid=46, filesize=11.7 K 2024-11-26T16:41:50,770 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 8c8b9347f011e2219af1b66f205e204f in 1264ms, sequenceid=46, compaction requested=false 2024-11-26T16:41:50,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:50,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:50,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-26T16:41:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-26T16:41:50,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-26T16:41:50,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3400 sec 2024-11-26T16:41:50,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.3530 sec 2024-11-26T16:41:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:50,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T16:41:50,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:41:50,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:50,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:41:50,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:50,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:41:50,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:50,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/06ad7e9e5935428a9052909b9d293538 is 50, key is test_row_0/A:col10/1732639310782/Put/seqid=0 2024-11-26T16:41:50,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742063_1239 (size=14341) 2024-11-26T16:41:50,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639370827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:50,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639370832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:50,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:50,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:50,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:50,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639370941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:50,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639370941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:51,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639371151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:51,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639371151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:51,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/06ad7e9e5935428a9052909b9d293538 2024-11-26T16:41:51,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/702a37b6b55e4b0daac7e389b6b4faa6 is 50, key is test_row_0/B:col10/1732639310782/Put/seqid=0 2024-11-26T16:41:51,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742064_1240 (size=12001) 2024-11-26T16:41:51,387 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T16:41:51,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639371461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:51,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:51,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:51,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639371461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:51,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:51,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:51,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:51,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:51,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/702a37b6b55e4b0daac7e389b6b4faa6 2024-11-26T16:41:51,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/947086e2e48c45be87d4903fff0e6afe is 50, key is test_row_0/C:col10/1732639310782/Put/seqid=0 2024-11-26T16:41:51,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742065_1241 (size=12001) 2024-11-26T16:41:51,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/947086e2e48c45be87d4903fff0e6afe 2024-11-26T16:41:51,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/06ad7e9e5935428a9052909b9d293538 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/06ad7e9e5935428a9052909b9d293538 2024-11-26T16:41:51,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/06ad7e9e5935428a9052909b9d293538, entries=200, sequenceid=59, filesize=14.0 K 2024-11-26T16:41:51,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/702a37b6b55e4b0daac7e389b6b4faa6 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/702a37b6b55e4b0daac7e389b6b4faa6 2024-11-26T16:41:51,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/702a37b6b55e4b0daac7e389b6b4faa6, entries=150, sequenceid=59, filesize=11.7 K 2024-11-26T16:41:51,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/947086e2e48c45be87d4903fff0e6afe as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/947086e2e48c45be87d4903fff0e6afe 2024-11-26T16:41:51,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/947086e2e48c45be87d4903fff0e6afe, entries=150, sequenceid=59, filesize=11.7 K 2024-11-26T16:41:51,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8c8b9347f011e2219af1b66f205e204f in 873ms, sequenceid=59, compaction requested=true 2024-11-26T16:41:51,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:51,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:41:51,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:51,656 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:51,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:41:51,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:51,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:41:51,656 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:51,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:51,657 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:51,657 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/A is initiating minor compaction (all files) 2024-11-26T16:41:51,657 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/A in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:51,657 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/0b795fd6e586474a8a955eb519e756b9, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/a9f75cc33df94d0d91e8a0519a0f173b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/06ad7e9e5935428a9052909b9d293538] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=37.4 K 2024-11-26T16:41:51,658 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:51,658 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b795fd6e586474a8a955eb519e756b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732639308432 2024-11-26T16:41:51,658 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/B is initiating minor compaction (all files) 2024-11-26T16:41:51,658 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/B in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:51,658 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a7c3d7e7722f4dbebb4a7198033ad0bc, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/86f6f86ce2d849da9cdc44ecb0094b7d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/702a37b6b55e4b0daac7e389b6b4faa6] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=35.2 K 2024-11-26T16:41:51,659 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a7c3d7e7722f4dbebb4a7198033ad0bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732639308432 2024-11-26T16:41:51,659 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9f75cc33df94d0d91e8a0519a0f173b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732639308480 2024-11-26T16:41:51,659 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06ad7e9e5935428a9052909b9d293538, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732639309629 2024-11-26T16:41:51,659 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 86f6f86ce2d849da9cdc44ecb0094b7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732639308480 2024-11-26T16:41:51,659 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 702a37b6b55e4b0daac7e389b6b4faa6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732639309632 2024-11-26T16:41:51,668 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#A#compaction#198 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:51,669 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/19093a1095a84263abdf35055b5091da is 50, key is test_row_0/A:col10/1732639310782/Put/seqid=0 2024-11-26T16:41:51,672 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#B#compaction#199 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:51,672 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/45d0bf36ca8247a2bd158fcc8a3c7345 is 50, key is test_row_0/B:col10/1732639310782/Put/seqid=0 2024-11-26T16:41:51,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742066_1242 (size=12104) 2024-11-26T16:41:51,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742067_1243 (size=12104) 2024-11-26T16:41:51,747 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/45d0bf36ca8247a2bd158fcc8a3c7345 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/45d0bf36ca8247a2bd158fcc8a3c7345 2024-11-26T16:41:51,756 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/B of 8c8b9347f011e2219af1b66f205e204f into 45d0bf36ca8247a2bd158fcc8a3c7345(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:51,756 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:51,756 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/B, priority=13, startTime=1732639311656; duration=0sec 2024-11-26T16:41:51,756 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:51,756 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:B 2024-11-26T16:41:51,756 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:51,758 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:51,758 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/C is initiating minor compaction (all files) 2024-11-26T16:41:51,758 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/C in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:51,758 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/a22304964d794101b872cbc50a16d386, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/3cf658b406fd43fbbed930d90560ca82, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/947086e2e48c45be87d4903fff0e6afe] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=35.2 K 2024-11-26T16:41:51,758 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a22304964d794101b872cbc50a16d386, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732639308432 2024-11-26T16:41:51,759 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cf658b406fd43fbbed930d90560ca82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732639308480 2024-11-26T16:41:51,759 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 947086e2e48c45be87d4903fff0e6afe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732639309632 2024-11-26T16:41:51,776 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#C#compaction#200 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:51,777 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/0f055f581bc840768a18e5d6c927f9d5 is 50, key is test_row_0/C:col10/1732639310782/Put/seqid=0 2024-11-26T16:41:51,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742068_1244 (size=12104) 2024-11-26T16:41:51,821 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/0f055f581bc840768a18e5d6c927f9d5 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/0f055f581bc840768a18e5d6c927f9d5 2024-11-26T16:41:51,828 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/C of 8c8b9347f011e2219af1b66f205e204f into 0f055f581bc840768a18e5d6c927f9d5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:51,828 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:51,828 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/C, priority=13, startTime=1732639311656; duration=0sec 2024-11-26T16:41:51,828 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:51,828 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:C 2024-11-26T16:41:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:51,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T16:41:51,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:41:51,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:41:51,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:41:51,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:41:51,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:41:51,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:41:51,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/6514e557cd4b4b499a6106b0124b8fe1 is 50, key is test_row_0/A:col10/1732639310818/Put/seqid=0 2024-11-26T16:41:51,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742069_1245 (size=14341) 2024-11-26T16:41:52,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:52,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639371998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:52,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:52,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:52,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639372002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:52,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:52,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639372112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:52,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:52,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639372112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:52,123 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/19093a1095a84263abdf35055b5091da as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/19093a1095a84263abdf35055b5091da 2024-11-26T16:41:52,138 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/A of 8c8b9347f011e2219af1b66f205e204f into 19093a1095a84263abdf35055b5091da(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:52,138 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:52,138 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/A, priority=13, startTime=1732639311656; duration=0sec 2024-11-26T16:41:52,138 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:52,138 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:A 2024-11-26T16:41:52,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639372321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:52,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639372321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:52,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:52,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:52,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/6514e557cd4b4b499a6106b0124b8fe1 2024-11-26T16:41:52,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/25faac0e441448a7b0ee48e2c4a6667d is 50, key is test_row_0/B:col10/1732639310818/Put/seqid=0 2024-11-26T16:41:52,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742070_1246 (size=12001) 2024-11-26T16:41:52,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/25faac0e441448a7b0ee48e2c4a6667d 2024-11-26T16:41:52,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/60bca55dbe7140019d49ec776e2a3538 is 50, key is test_row_0/C:col10/1732639310818/Put/seqid=0 2024-11-26T16:41:52,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742071_1247 (size=12001) 2024-11-26T16:41:52,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/60bca55dbe7140019d49ec776e2a3538 2024-11-26T16:41:52,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/6514e557cd4b4b499a6106b0124b8fe1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/6514e557cd4b4b499a6106b0124b8fe1 2024-11-26T16:41:52,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/6514e557cd4b4b499a6106b0124b8fe1, entries=200, sequenceid=82, filesize=14.0 K 2024-11-26T16:41:52,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/25faac0e441448a7b0ee48e2c4a6667d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/25faac0e441448a7b0ee48e2c4a6667d 2024-11-26T16:41:52,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/25faac0e441448a7b0ee48e2c4a6667d, entries=150, sequenceid=82, filesize=11.7 K 2024-11-26T16:41:52,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/60bca55dbe7140019d49ec776e2a3538 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/60bca55dbe7140019d49ec776e2a3538 2024-11-26T16:41:52,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/60bca55dbe7140019d49ec776e2a3538, entries=150, sequenceid=82, filesize=11.7 K 2024-11-26T16:41:52,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 8c8b9347f011e2219af1b66f205e204f in 517ms, sequenceid=82, compaction requested=false 2024-11-26T16:41:52,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-26T16:41:52,582 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:52,583 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:41:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-26T16:41:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T16:41:52,586 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:41:52,587 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:41:52,587 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:41:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:52,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T16:41:52,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:41:52,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:52,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:41:52,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:52,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:41:52,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:52,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/cf544b2efa354fe09e0918e3275d94b6 is 50, key is test_row_0/A:col10/1732639312636/Put/seqid=0 2024-11-26T16:41:52,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742072_1248 (size=14341) 2024-11-26T16:41:52,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/cf544b2efa354fe09e0918e3275d94b6 2024-11-26T16:41:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T16:41:52,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/19b0d0e8c7f847d1b90aa46101d0996b is 50, key is test_row_0/B:col10/1732639312636/Put/seqid=0 2024-11-26T16:41:52,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742073_1249 (size=12001) 2024-11-26T16:41:52,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T16:41:52,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:52,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:52,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:52,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:52,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:52,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639372701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:52,743 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in 8c8b9347f011e2219af1b66f205e204f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:41:52,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:52,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639372706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:52,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:52,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639372736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639372736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:52,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639372742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:52,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:52,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:52,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:52,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:52,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639372851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:52,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:52,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639372851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:52,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:52,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:52,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639372851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639372852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:52,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T16:41:52,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:52,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:52,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:52,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:52,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T16:41:53,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T16:41:53,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:53,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:53,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:53,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:53,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:53,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:53,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639373061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:53,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639373061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:53,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639373061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:53,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639373061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:53,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/19b0d0e8c7f847d1b90aa46101d0996b 2024-11-26T16:41:53,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/df1c0e5982de49be9bd64926a417192e is 50, key is test_row_0/C:col10/1732639312636/Put/seqid=0 2024-11-26T16:41:53,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742074_1250 (size=12001) 2024-11-26T16:41:53,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/df1c0e5982de49be9bd64926a417192e 2024-11-26T16:41:53,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/cf544b2efa354fe09e0918e3275d94b6 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/cf544b2efa354fe09e0918e3275d94b6 2024-11-26T16:41:53,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/cf544b2efa354fe09e0918e3275d94b6, entries=200, sequenceid=96, filesize=14.0 K 2024-11-26T16:41:53,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/19b0d0e8c7f847d1b90aa46101d0996b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/19b0d0e8c7f847d1b90aa46101d0996b 2024-11-26T16:41:53,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/19b0d0e8c7f847d1b90aa46101d0996b, entries=150, sequenceid=96, filesize=11.7 K 2024-11-26T16:41:53,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/df1c0e5982de49be9bd64926a417192e as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/df1c0e5982de49be9bd64926a417192e 2024-11-26T16:41:53,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/df1c0e5982de49be9bd64926a417192e, entries=150, sequenceid=96, filesize=11.7 K 2024-11-26T16:41:53,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 8c8b9347f011e2219af1b66f205e204f in 520ms, sequenceid=96, compaction requested=true 2024-11-26T16:41:53,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:53,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:41:53,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:53,158 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:53,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:41:53,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:53,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:41:53,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:53,158 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:53,159 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:53,159 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:53,160 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/A is initiating minor compaction (all files) 2024-11-26T16:41:53,160 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/B is initiating minor compaction (all files) 2024-11-26T16:41:53,160 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/A in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:53,160 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/B in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:53,160 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/19093a1095a84263abdf35055b5091da, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/6514e557cd4b4b499a6106b0124b8fe1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/cf544b2efa354fe09e0918e3275d94b6] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=39.8 K 2024-11-26T16:41:53,160 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/45d0bf36ca8247a2bd158fcc8a3c7345, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/25faac0e441448a7b0ee48e2c4a6667d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/19b0d0e8c7f847d1b90aa46101d0996b] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=35.3 K 2024-11-26T16:41:53,160 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19093a1095a84263abdf35055b5091da, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732639309632 2024-11-26T16:41:53,160 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 45d0bf36ca8247a2bd158fcc8a3c7345, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732639309632 2024-11-26T16:41:53,161 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 25faac0e441448a7b0ee48e2c4a6667d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732639310818 2024-11-26T16:41:53,161 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6514e557cd4b4b499a6106b0124b8fe1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732639310818 2024-11-26T16:41:53,161 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 19b0d0e8c7f847d1b90aa46101d0996b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732639311987 2024-11-26T16:41:53,161 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf544b2efa354fe09e0918e3275d94b6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732639311987 2024-11-26T16:41:53,171 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#B#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:53,172 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/46f161bfe85d461e9ae896e6cdaad0d1 is 50, key is test_row_0/B:col10/1732639312636/Put/seqid=0 2024-11-26T16:41:53,175 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#A#compaction#208 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:53,175 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/514bb8df494845d98f5f37b465bc44fe is 50, key is test_row_0/A:col10/1732639312636/Put/seqid=0 2024-11-26T16:41:53,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742075_1251 (size=12207) 2024-11-26T16:41:53,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742076_1252 (size=12207) 2024-11-26T16:41:53,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-26T16:41:53,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:53,202 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T16:41:53,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:41:53,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:53,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:41:53,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:53,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:41:53,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:53,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/142ad18243a441b992fdfc9abc0a9b9e is 50, key is test_row_0/A:col10/1732639312676/Put/seqid=0 2024-11-26T16:41:53,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T16:41:53,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742077_1253 (size=12001) 2024-11-26T16:41:53,221 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/142ad18243a441b992fdfc9abc0a9b9e 2024-11-26T16:41:53,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/f8cf776b7e4d4675afe60c0703c2963a is 50, key is test_row_0/B:col10/1732639312676/Put/seqid=0 2024-11-26T16:41:53,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742078_1254 (size=12001) 2024-11-26T16:41:53,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:53,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:53,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:53,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639373385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:53,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:53,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639373390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:53,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639373390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:53,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:53,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639373391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:53,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:53,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639373501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:53,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639373501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:53,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:53,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639373501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:53,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639373501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:53,588 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/46f161bfe85d461e9ae896e6cdaad0d1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/46f161bfe85d461e9ae896e6cdaad0d1 2024-11-26T16:41:53,594 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/B of 8c8b9347f011e2219af1b66f205e204f into 46f161bfe85d461e9ae896e6cdaad0d1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:53,594 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:53,594 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/B, priority=13, startTime=1732639313158; duration=0sec 2024-11-26T16:41:53,594 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:53,594 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:B 2024-11-26T16:41:53,594 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:53,595 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:53,595 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/514bb8df494845d98f5f37b465bc44fe as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/514bb8df494845d98f5f37b465bc44fe 2024-11-26T16:41:53,595 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/C is initiating minor compaction (all files) 2024-11-26T16:41:53,595 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/C in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:53,595 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/0f055f581bc840768a18e5d6c927f9d5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/60bca55dbe7140019d49ec776e2a3538, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/df1c0e5982de49be9bd64926a417192e] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=35.3 K 2024-11-26T16:41:53,596 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f055f581bc840768a18e5d6c927f9d5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732639309632 2024-11-26T16:41:53,596 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 60bca55dbe7140019d49ec776e2a3538, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732639310818 2024-11-26T16:41:53,597 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting df1c0e5982de49be9bd64926a417192e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732639311987 2024-11-26T16:41:53,601 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/A of 8c8b9347f011e2219af1b66f205e204f into 514bb8df494845d98f5f37b465bc44fe(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:53,601 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:53,601 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/A, priority=13, startTime=1732639313158; duration=0sec 2024-11-26T16:41:53,601 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:53,601 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:A 2024-11-26T16:41:53,605 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#C#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:53,606 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/de09fc821e1c46c4846e63a6d89d81c8 is 50, key is test_row_0/C:col10/1732639312636/Put/seqid=0 2024-11-26T16:41:53,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742079_1255 (size=12207) 2024-11-26T16:41:53,639 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/f8cf776b7e4d4675afe60c0703c2963a 2024-11-26T16:41:53,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/89c206f5dc594ac092a0294daeb21c7a is 50, key is test_row_0/C:col10/1732639312676/Put/seqid=0 2024-11-26T16:41:53,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742080_1256 (size=12001) 2024-11-26T16:41:53,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:53,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639373711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:53,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:53,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639373711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:53,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:53,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639373711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:53,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:53,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639373711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:53,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:53,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:53,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T16:41:54,019 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/de09fc821e1c46c4846e63a6d89d81c8 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/de09fc821e1c46c4846e63a6d89d81c8 2024-11-26T16:41:54,025 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/C of 8c8b9347f011e2219af1b66f205e204f into de09fc821e1c46c4846e63a6d89d81c8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:54,025 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:54,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:54,025 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/C, priority=13, startTime=1732639313158; duration=0sec 2024-11-26T16:41:54,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639374021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:54,025 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:54,025 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:C 2024-11-26T16:41:54,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639374021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:54,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639374021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:54,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639374021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:54,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:54,059 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/89c206f5dc594ac092a0294daeb21c7a 2024-11-26T16:41:54,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/142ad18243a441b992fdfc9abc0a9b9e as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/142ad18243a441b992fdfc9abc0a9b9e 2024-11-26T16:41:54,069 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/142ad18243a441b992fdfc9abc0a9b9e, entries=150, sequenceid=116, filesize=11.7 K 2024-11-26T16:41:54,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/f8cf776b7e4d4675afe60c0703c2963a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/f8cf776b7e4d4675afe60c0703c2963a 2024-11-26T16:41:54,073 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/f8cf776b7e4d4675afe60c0703c2963a, entries=150, sequenceid=116, filesize=11.7 K 2024-11-26T16:41:54,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/89c206f5dc594ac092a0294daeb21c7a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/89c206f5dc594ac092a0294daeb21c7a 2024-11-26T16:41:54,078 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/89c206f5dc594ac092a0294daeb21c7a, entries=150, sequenceid=116, filesize=11.7 K 2024-11-26T16:41:54,079 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=73.80 KB/75570 for 8c8b9347f011e2219af1b66f205e204f in 877ms, sequenceid=116, compaction requested=false 2024-11-26T16:41:54,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:54,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:54,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-26T16:41:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-26T16:41:54,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-26T16:41:54,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4940 sec 2024-11-26T16:41:54,085 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.5010 sec 2024-11-26T16:41:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:54,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T16:41:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:41:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:41:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:41:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:54,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/317218ac3e344aea8a6dc8e676792d50 is 50, key is test_row_0/A:col10/1732639314532/Put/seqid=0 2024-11-26T16:41:54,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742081_1257 (size=14441) 2024-11-26T16:41:54,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/317218ac3e344aea8a6dc8e676792d50 2024-11-26T16:41:54,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/a9f33e211fe2440ebdd79c68bb6893ab is 50, key is test_row_0/B:col10/1732639314532/Put/seqid=0 2024-11-26T16:41:54,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639374553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:54,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639374554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:54,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:54,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639374555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:54,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:54,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742082_1258 (size=12051) 2024-11-26T16:41:54,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:54,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639374560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:54,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639374671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:54,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:54,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639374671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:54,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:54,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639374671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:54,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639374671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-26T16:41:54,731 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:54,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:41:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-26T16:41:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T16:41:54,733 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:41:54,734 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:41:54,734 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:41:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T16:41:54,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639374881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:54,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639374881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:54,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:54,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639374881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:54,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:54,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639374881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:54,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-26T16:41:54,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:54,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:54,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:54,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:54,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:54,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/a9f33e211fe2440ebdd79c68bb6893ab 2024-11-26T16:41:54,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/49766e5ff9004220be419f0c14c5c9af is 50, key is test_row_0/C:col10/1732639314532/Put/seqid=0 2024-11-26T16:41:54,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742083_1259 (size=12051) 2024-11-26T16:41:55,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-26T16:41:55,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:55,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:55,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:55,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:55,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T16:41:55,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-26T16:41:55,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:55,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:55,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:55,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:55,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:55,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:55,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639375191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:55,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:55,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:55,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639375191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:55,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:55,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639375192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639375192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:55,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-26T16:41:55,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:55,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:55,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:55,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:55,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T16:41:55,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/49766e5ff9004220be419f0c14c5c9af 2024-11-26T16:41:55,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/317218ac3e344aea8a6dc8e676792d50 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/317218ac3e344aea8a6dc8e676792d50 2024-11-26T16:41:55,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/317218ac3e344aea8a6dc8e676792d50, entries=200, sequenceid=134, filesize=14.1 K 2024-11-26T16:41:55,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/a9f33e211fe2440ebdd79c68bb6893ab as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a9f33e211fe2440ebdd79c68bb6893ab 2024-11-26T16:41:55,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a9f33e211fe2440ebdd79c68bb6893ab, entries=150, sequenceid=134, filesize=11.8 K 2024-11-26T16:41:55,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/49766e5ff9004220be419f0c14c5c9af as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/49766e5ff9004220be419f0c14c5c9af 2024-11-26T16:41:55,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/49766e5ff9004220be419f0c14c5c9af, entries=150, sequenceid=134, filesize=11.8 K 2024-11-26T16:41:55,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 8c8b9347f011e2219af1b66f205e204f in 864ms, sequenceid=134, compaction requested=true 2024-11-26T16:41:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:41:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:41:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:55,398 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:41:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T16:41:55,399 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:55,399 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:55,399 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/A is initiating minor compaction (all files) 2024-11-26T16:41:55,400 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/A in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:55,400 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/514bb8df494845d98f5f37b465bc44fe, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/142ad18243a441b992fdfc9abc0a9b9e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/317218ac3e344aea8a6dc8e676792d50] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=37.7 K 2024-11-26T16:41:55,400 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 514bb8df494845d98f5f37b465bc44fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732639311987 2024-11-26T16:41:55,401 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:55,401 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 142ad18243a441b992fdfc9abc0a9b9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732639312676 2024-11-26T16:41:55,401 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/B is initiating minor compaction (all files) 2024-11-26T16:41:55,401 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/B in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:55,401 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/46f161bfe85d461e9ae896e6cdaad0d1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/f8cf776b7e4d4675afe60c0703c2963a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a9f33e211fe2440ebdd79c68bb6893ab] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=35.4 K 2024-11-26T16:41:55,401 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 317218ac3e344aea8a6dc8e676792d50, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732639313381 2024-11-26T16:41:55,401 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 46f161bfe85d461e9ae896e6cdaad0d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732639311987 2024-11-26T16:41:55,402 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting f8cf776b7e4d4675afe60c0703c2963a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732639312676 2024-11-26T16:41:55,402 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a9f33e211fe2440ebdd79c68bb6893ab, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732639313381 2024-11-26T16:41:55,410 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#A#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:55,410 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/53c1bb8419c6454b8f9ef05dd54f2138 is 50, key is test_row_0/A:col10/1732639314532/Put/seqid=0 2024-11-26T16:41:55,410 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#B#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:55,411 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/59249f6d849a4e148940a508628b632a is 50, key is test_row_0/B:col10/1732639314532/Put/seqid=0 2024-11-26T16:41:55,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742085_1261 (size=12359) 2024-11-26T16:41:55,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742084_1260 (size=12359) 2024-11-26T16:41:55,431 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/53c1bb8419c6454b8f9ef05dd54f2138 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/53c1bb8419c6454b8f9ef05dd54f2138 2024-11-26T16:41:55,437 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/A of 8c8b9347f011e2219af1b66f205e204f into 53c1bb8419c6454b8f9ef05dd54f2138(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:55,437 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:55,437 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/A, priority=13, startTime=1732639315398; duration=0sec 2024-11-26T16:41:55,437 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:55,437 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:A 2024-11-26T16:41:55,437 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:55,439 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:55,439 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/C is initiating minor compaction (all files) 2024-11-26T16:41:55,439 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/C in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:55,439 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/de09fc821e1c46c4846e63a6d89d81c8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/89c206f5dc594ac092a0294daeb21c7a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/49766e5ff9004220be419f0c14c5c9af] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=35.4 K 2024-11-26T16:41:55,440 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting de09fc821e1c46c4846e63a6d89d81c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732639311987 2024-11-26T16:41:55,441 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89c206f5dc594ac092a0294daeb21c7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732639312676 2024-11-26T16:41:55,441 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49766e5ff9004220be419f0c14c5c9af, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732639313381 2024-11-26T16:41:55,453 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#C#compaction#218 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:55,454 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/8ad0c6dfd06247c580f711c43ef7aac5 is 50, key is test_row_0/C:col10/1732639314532/Put/seqid=0 2024-11-26T16:41:55,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742086_1262 (size=12359) 2024-11-26T16:41:55,491 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/8ad0c6dfd06247c580f711c43ef7aac5 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/8ad0c6dfd06247c580f711c43ef7aac5 2024-11-26T16:41:55,499 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/C of 8c8b9347f011e2219af1b66f205e204f into 8ad0c6dfd06247c580f711c43ef7aac5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:55,499 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:55,499 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/C, priority=13, startTime=1732639315398; duration=0sec 2024-11-26T16:41:55,499 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:55,499 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:C 2024-11-26T16:41:55,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-26T16:41:55,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:55,501 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-26T16:41:55,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:41:55,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:55,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:41:55,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:55,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:41:55,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:55,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/20cf12b8fa5d4b18bd67822453d6a534 is 50, key is test_row_0/A:col10/1732639314554/Put/seqid=0 2024-11-26T16:41:55,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742087_1263 (size=12151) 2024-11-26T16:41:55,515 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/20cf12b8fa5d4b18bd67822453d6a534 2024-11-26T16:41:55,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/4945799070244047b7d7a4cb248c2d71 is 50, key is test_row_0/B:col10/1732639314554/Put/seqid=0 2024-11-26T16:41:55,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742088_1264 (size=12151) 2024-11-26T16:41:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:55,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:55,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639375718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:55,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:55,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639375723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:55,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:55,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639375724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:55,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:55,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:55,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639375725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:55,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:55,824 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/59249f6d849a4e148940a508628b632a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/59249f6d849a4e148940a508628b632a 2024-11-26T16:41:55,829 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/B of 8c8b9347f011e2219af1b66f205e204f into 59249f6d849a4e148940a508628b632a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:55,829 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:55,829 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/B, priority=13, startTime=1732639315398; duration=0sec 2024-11-26T16:41:55,829 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:55,829 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:B 2024-11-26T16:41:55,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639375831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:55,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:55,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639375832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:55,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:55,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639375832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:55,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:55,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639375841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:55,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:55,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T16:41:55,931 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/4945799070244047b7d7a4cb248c2d71 2024-11-26T16:41:55,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/903ac39919c94c66b72cce91d2743928 is 50, key is test_row_0/C:col10/1732639314554/Put/seqid=0 2024-11-26T16:41:55,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742089_1265 (size=12151) 2024-11-26T16:41:56,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639376041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:56,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:56,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639376041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:56,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639376041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:56,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639376051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:56,354 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/903ac39919c94c66b72cce91d2743928 2024-11-26T16:41:56,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639376351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:56,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639376352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:56,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639376352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:56,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/20cf12b8fa5d4b18bd67822453d6a534 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/20cf12b8fa5d4b18bd67822453d6a534 2024-11-26T16:41:56,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639376361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:56,365 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/20cf12b8fa5d4b18bd67822453d6a534, entries=150, sequenceid=153, filesize=11.9 K 2024-11-26T16:41:56,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/4945799070244047b7d7a4cb248c2d71 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/4945799070244047b7d7a4cb248c2d71 2024-11-26T16:41:56,371 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/4945799070244047b7d7a4cb248c2d71, entries=150, sequenceid=153, filesize=11.9 K 2024-11-26T16:41:56,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/903ac39919c94c66b72cce91d2743928 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/903ac39919c94c66b72cce91d2743928 2024-11-26T16:41:56,376 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/903ac39919c94c66b72cce91d2743928, entries=150, sequenceid=153, filesize=11.9 K 2024-11-26T16:41:56,377 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 8c8b9347f011e2219af1b66f205e204f in 876ms, sequenceid=153, compaction requested=false 2024-11-26T16:41:56,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:56,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:56,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-26T16:41:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-26T16:41:56,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-26T16:41:56,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6450 sec 2024-11-26T16:41:56,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.6500 sec 2024-11-26T16:41:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:56,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T16:41:56,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:41:56,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:56,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:41:56,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:56,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:41:56,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:56,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/7a568514bb39448ca581f9b9a7c82462 is 50, key is test_row_0/A:col10/1732639315717/Put/seqid=0 2024-11-26T16:41:56,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742090_1266 (size=12151) 2024-11-26T16:41:56,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:56,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639376846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:56,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:56,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:56,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639376861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:56,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:56,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639376861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:56,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:56,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:56,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639376871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:56,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639376871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:56,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-26T16:41:56,881 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:56,881 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:41:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-26T16:41:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T16:41:56,884 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:41:56,884 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:41:56,884 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:41:56,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639376961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T16:41:57,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T16:41:57,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:57,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:57,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:57,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639377171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:57,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:57,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:57,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:57,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T16:41:57,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:57,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:57,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:57,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/7a568514bb39448ca581f9b9a7c82462 2024-11-26T16:41:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T16:41:57,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/4379fb32d4e04eb6800dbf04bd8e118b is 50, key is test_row_0/B:col10/1732639315717/Put/seqid=0 2024-11-26T16:41:57,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742091_1267 (size=12151) 2024-11-26T16:41:57,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T16:41:57,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:57,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:57,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:57,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639377481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:57,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:57,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:57,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:57,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T16:41:57,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:57,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:57,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:57,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T16:41:57,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/4379fb32d4e04eb6800dbf04bd8e118b 2024-11-26T16:41:57,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/5fed311d528842e28381628e8d4620b5 is 50, key is test_row_0/C:col10/1732639315717/Put/seqid=0 2024-11-26T16:41:57,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742092_1268 (size=12151) 2024-11-26T16:41:57,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T16:41:57,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:57,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:57,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:57,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T16:41:57,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:57,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:57,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:57,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639377871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:57,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639377871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:57,875 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in 8c8b9347f011e2219af1b66f205e204f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:41:57,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:57,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:57,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:57,875 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in 8c8b9347f011e2219af1b66f205e204f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:41:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:57,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:57,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639377881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:57,885 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in 8c8b9347f011e2219af1b66f205e204f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:41:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:57,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:57,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:57,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639377891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:57,896 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_0' in 8c8b9347f011e2219af1b66f205e204f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:41:57,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:57,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:57,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:57,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T16:41:57,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:57,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:57,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:57,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:57,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:57,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639377991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:57,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:57,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:57,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:58,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T16:41:58,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/5fed311d528842e28381628e8d4620b5 2024-11-26T16:41:58,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/7a568514bb39448ca581f9b9a7c82462 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/7a568514bb39448ca581f9b9a7c82462 2024-11-26T16:41:58,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/7a568514bb39448ca581f9b9a7c82462, entries=150, sequenceid=170, filesize=11.9 K 2024-11-26T16:41:58,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/4379fb32d4e04eb6800dbf04bd8e118b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/4379fb32d4e04eb6800dbf04bd8e118b 2024-11-26T16:41:58,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/4379fb32d4e04eb6800dbf04bd8e118b, entries=150, sequenceid=170, filesize=11.9 K 2024-11-26T16:41:58,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/5fed311d528842e28381628e8d4620b5 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/5fed311d528842e28381628e8d4620b5 2024-11-26T16:41:58,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/5fed311d528842e28381628e8d4620b5, entries=150, sequenceid=170, filesize=11.9 K 2024-11-26T16:41:58,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 8c8b9347f011e2219af1b66f205e204f in 1264ms, sequenceid=170, compaction requested=true 2024-11-26T16:41:58,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:58,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:41:58,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:58,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:41:58,048 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:58,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:58,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:41:58,048 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:58,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:58,049 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:58,049 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:58,049 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/A is initiating minor compaction (all files) 2024-11-26T16:41:58,049 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/B is initiating minor compaction (all files) 2024-11-26T16:41:58,049 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/A in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:58,049 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/53c1bb8419c6454b8f9ef05dd54f2138, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/20cf12b8fa5d4b18bd67822453d6a534, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/7a568514bb39448ca581f9b9a7c82462] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=35.8 K 2024-11-26T16:41:58,049 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/B in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:58,050 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/59249f6d849a4e148940a508628b632a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/4945799070244047b7d7a4cb248c2d71, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/4379fb32d4e04eb6800dbf04bd8e118b] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=35.8 K 2024-11-26T16:41:58,050 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53c1bb8419c6454b8f9ef05dd54f2138, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732639313381 2024-11-26T16:41:58,050 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 59249f6d849a4e148940a508628b632a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732639313381 2024-11-26T16:41:58,051 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20cf12b8fa5d4b18bd67822453d6a534, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732639314539 2024-11-26T16:41:58,051 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 4945799070244047b7d7a4cb248c2d71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732639314539 2024-11-26T16:41:58,051 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a568514bb39448ca581f9b9a7c82462, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732639315717 2024-11-26T16:41:58,051 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 4379fb32d4e04eb6800dbf04bd8e118b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732639315717 2024-11-26T16:41:58,060 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#A#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:58,061 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/b652e53f9f9041bdae2d65111edfdd25 is 50, key is test_row_0/A:col10/1732639315717/Put/seqid=0 2024-11-26T16:41:58,062 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#B#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:58,063 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/5c2a48bdfc6047fd8e2431818dad44fa is 50, key is test_row_0/B:col10/1732639315717/Put/seqid=0 2024-11-26T16:41:58,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742093_1269 (size=12561) 2024-11-26T16:41:58,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742094_1270 (size=12561) 2024-11-26T16:41:58,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-26T16:41:58,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:58,111 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-26T16:41:58,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:41:58,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:58,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:41:58,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:58,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:41:58,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:58,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/7a369f9a0277496fa5d1e1c183b95819 is 50, key is test_row_0/A:col10/1732639316839/Put/seqid=0 2024-11-26T16:41:58,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742095_1271 (size=12151) 2024-11-26T16:41:58,473 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/b652e53f9f9041bdae2d65111edfdd25 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/b652e53f9f9041bdae2d65111edfdd25 2024-11-26T16:41:58,477 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/5c2a48bdfc6047fd8e2431818dad44fa as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/5c2a48bdfc6047fd8e2431818dad44fa 2024-11-26T16:41:58,481 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/A of 8c8b9347f011e2219af1b66f205e204f into b652e53f9f9041bdae2d65111edfdd25(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:58,481 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:58,481 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/A, priority=13, startTime=1732639318048; duration=0sec 2024-11-26T16:41:58,481 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:58,481 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:A 2024-11-26T16:41:58,482 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:58,482 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/B of 8c8b9347f011e2219af1b66f205e204f into 5c2a48bdfc6047fd8e2431818dad44fa(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:58,482 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:58,482 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/B, priority=13, startTime=1732639318048; duration=0sec 2024-11-26T16:41:58,482 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:58,482 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:B 2024-11-26T16:41:58,483 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:58,483 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/C is initiating minor compaction (all files) 2024-11-26T16:41:58,483 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/C in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:58,483 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/8ad0c6dfd06247c580f711c43ef7aac5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/903ac39919c94c66b72cce91d2743928, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/5fed311d528842e28381628e8d4620b5] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=35.8 K 2024-11-26T16:41:58,484 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ad0c6dfd06247c580f711c43ef7aac5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732639313381 2024-11-26T16:41:58,484 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 903ac39919c94c66b72cce91d2743928, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732639314539 2024-11-26T16:41:58,484 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fed311d528842e28381628e8d4620b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732639315717 2024-11-26T16:41:58,493 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#C#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:58,494 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/5eabfe38d73441a3a29762de22b5c9a7 is 50, key is test_row_0/C:col10/1732639315717/Put/seqid=0 2024-11-26T16:41:58,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742096_1272 (size=12561) 2024-11-26T16:41:58,521 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/7a369f9a0277496fa5d1e1c183b95819 2024-11-26T16:41:58,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/fc2947dadc924ca1955b744eb47ccc21 is 50, key is test_row_0/B:col10/1732639316839/Put/seqid=0 2024-11-26T16:41:58,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742097_1273 (size=12151) 2024-11-26T16:41:58,911 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/5eabfe38d73441a3a29762de22b5c9a7 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/5eabfe38d73441a3a29762de22b5c9a7 2024-11-26T16:41:58,918 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/C of 8c8b9347f011e2219af1b66f205e204f into 5eabfe38d73441a3a29762de22b5c9a7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:58,918 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:58,918 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/C, priority=13, startTime=1732639318048; duration=0sec 2024-11-26T16:41:58,918 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:58,918 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:C 2024-11-26T16:41:58,936 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/fc2947dadc924ca1955b744eb47ccc21 2024-11-26T16:41:58,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/aad9b71bb1654bf591b993960311dc2b is 50, key is test_row_0/C:col10/1732639316839/Put/seqid=0 2024-11-26T16:41:58,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742098_1274 (size=12151) 2024-11-26T16:41:58,950 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/aad9b71bb1654bf591b993960311dc2b 2024-11-26T16:41:58,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/7a369f9a0277496fa5d1e1c183b95819 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/7a369f9a0277496fa5d1e1c183b95819 2024-11-26T16:41:58,959 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/7a369f9a0277496fa5d1e1c183b95819, entries=150, sequenceid=186, filesize=11.9 K 2024-11-26T16:41:58,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/fc2947dadc924ca1955b744eb47ccc21 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/fc2947dadc924ca1955b744eb47ccc21 2024-11-26T16:41:58,965 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/fc2947dadc924ca1955b744eb47ccc21, entries=150, sequenceid=186, filesize=11.9 K 2024-11-26T16:41:58,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/aad9b71bb1654bf591b993960311dc2b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/aad9b71bb1654bf591b993960311dc2b 2024-11-26T16:41:58,971 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/aad9b71bb1654bf591b993960311dc2b, entries=150, sequenceid=186, filesize=11.9 K 2024-11-26T16:41:58,972 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 8c8b9347f011e2219af1b66f205e204f in 861ms, sequenceid=186, compaction requested=false 2024-11-26T16:41:58,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:58,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:58,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-26T16:41:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-26T16:41:58,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-26T16:41:58,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0900 sec 2024-11-26T16:41:58,979 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.0960 sec 2024-11-26T16:41:59,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:59,019 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T16:41:59,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:41:59,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:59,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:41:59,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:59,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:41:59,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:41:59,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/6d1d66bd70b34e139253ae3ae7b8ab8c is 50, key is test_row_0/A:col10/1732639319016/Put/seqid=0 2024-11-26T16:41:59,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742099_1275 (size=12151) 2024-11-26T16:41:59,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-26T16:41:59,031 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:41:59,032 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:41:59,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-26T16:41:59,034 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:41:59,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T16:41:59,034 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:41:59,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:41:59,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639319073, max timestamp=1732639319105Num uniques 859; 2024-11-26T16:41:59,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:41:59,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639319073, max timestamp=1732639319105Num uniques 859; 2024-11-26T16:41:59,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:41:59,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639319073, max timestamp=1732639319105Num uniques 859; 2024-11-26T16:41:59,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:41:59,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T16:41:59,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T16:41:59,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:59,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:59,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:59,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:59,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:59,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:59,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639379217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:59,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:59,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639379331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:59,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:59,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:59,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:59,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T16:41:59,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:59,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:59,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:59,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:59,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:59,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T16:41:59,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/6d1d66bd70b34e139253ae3ae7b8ab8c 2024-11-26T16:41:59,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/dd86959c984649a5b65178ab20bb6c72 is 50, key is test_row_0/B:col10/1732639319016/Put/seqid=0 2024-11-26T16:41:59,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742100_1276 (size=12151) 2024-11-26T16:41:59,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T16:41:59,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:59,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:59,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:59,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:59,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:59,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639379541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:59,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:59,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:59,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:59,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T16:41:59,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:59,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:59,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:59,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:59,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:59,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:59,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T16:41:59,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T16:41:59,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:59,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:59,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:59,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:59,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:59,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:59,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/dd86959c984649a5b65178ab20bb6c72 2024-11-26T16:41:59,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/3498596f87d14418a29e2fe21d9787e9 is 50, key is test_row_0/C:col10/1732639319016/Put/seqid=0 2024-11-26T16:41:59,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742101_1277 (size=12151) 2024-11-26T16:41:59,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/3498596f87d14418a29e2fe21d9787e9 2024-11-26T16:41:59,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639379852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:59,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/6d1d66bd70b34e139253ae3ae7b8ab8c as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/6d1d66bd70b34e139253ae3ae7b8ab8c 2024-11-26T16:41:59,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/6d1d66bd70b34e139253ae3ae7b8ab8c, entries=150, sequenceid=199, filesize=11.9 K 2024-11-26T16:41:59,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/dd86959c984649a5b65178ab20bb6c72 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/dd86959c984649a5b65178ab20bb6c72 2024-11-26T16:41:59,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/dd86959c984649a5b65178ab20bb6c72, entries=150, sequenceid=199, filesize=11.9 K 2024-11-26T16:41:59,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/3498596f87d14418a29e2fe21d9787e9 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/3498596f87d14418a29e2fe21d9787e9 2024-11-26T16:41:59,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/3498596f87d14418a29e2fe21d9787e9, entries=150, sequenceid=199, filesize=11.9 K 2024-11-26T16:41:59,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 8c8b9347f011e2219af1b66f205e204f in 858ms, sequenceid=199, compaction requested=true 2024-11-26T16:41:59,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:59,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:41:59,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:59,876 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:59,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:41:59,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:59,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:41:59,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:59,876 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:59,877 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:59,877 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:59,877 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/B is initiating minor compaction (all files) 2024-11-26T16:41:59,877 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/A is initiating minor compaction (all files) 2024-11-26T16:41:59,877 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/B in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:59,877 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/A in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:59,877 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/5c2a48bdfc6047fd8e2431818dad44fa, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/fc2947dadc924ca1955b744eb47ccc21, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/dd86959c984649a5b65178ab20bb6c72] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=36.0 K 2024-11-26T16:41:59,877 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/b652e53f9f9041bdae2d65111edfdd25, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/7a369f9a0277496fa5d1e1c183b95819, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/6d1d66bd70b34e139253ae3ae7b8ab8c] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=36.0 K 2024-11-26T16:41:59,878 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting b652e53f9f9041bdae2d65111edfdd25, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732639315717 2024-11-26T16:41:59,878 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c2a48bdfc6047fd8e2431818dad44fa, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732639315717 2024-11-26T16:41:59,878 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting fc2947dadc924ca1955b744eb47ccc21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732639316817 2024-11-26T16:41:59,878 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a369f9a0277496fa5d1e1c183b95819, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732639316817 2024-11-26T16:41:59,879 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting dd86959c984649a5b65178ab20bb6c72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732639319010 2024-11-26T16:41:59,879 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d1d66bd70b34e139253ae3ae7b8ab8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732639319010 2024-11-26T16:41:59,887 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#B#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:59,887 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#A#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:59,887 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/0aeee3db914744939f129e98fe85803a is 50, key is test_row_0/B:col10/1732639319016/Put/seqid=0 2024-11-26T16:41:59,887 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/5cbd650499a9488ba8a79f763906e620 is 50, key is test_row_0/A:col10/1732639319016/Put/seqid=0 2024-11-26T16:41:59,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742102_1278 (size=12663) 2024-11-26T16:41:59,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742103_1279 (size=12663) 2024-11-26T16:41:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:41:59,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-26T16:41:59,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:41:59,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:41:59,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:41:59,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:41:59,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:41:59,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:41:59,913 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/0aeee3db914744939f129e98fe85803a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/0aeee3db914744939f129e98fe85803a 2024-11-26T16:41:59,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/4a229a99a2e940e6aeb5547af437c9cc is 50, key is test_row_0/A:col10/1732639319207/Put/seqid=0 2024-11-26T16:41:59,914 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/5cbd650499a9488ba8a79f763906e620 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/5cbd650499a9488ba8a79f763906e620 2024-11-26T16:41:59,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742104_1280 (size=12151) 2024-11-26T16:41:59,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639379914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:59,922 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/B of 8c8b9347f011e2219af1b66f205e204f into 0aeee3db914744939f129e98fe85803a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:59,922 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:59,922 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/B, priority=13, startTime=1732639319876; duration=0sec 2024-11-26T16:41:59,922 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:41:59,922 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:B 2024-11-26T16:41:59,922 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:41:59,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/4a229a99a2e940e6aeb5547af437c9cc 2024-11-26T16:41:59,923 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/A of 8c8b9347f011e2219af1b66f205e204f into 5cbd650499a9488ba8a79f763906e620(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:41:59,923 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:41:59,923 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/A, priority=13, startTime=1732639319876; duration=0sec 2024-11-26T16:41:59,923 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:41:59,923 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:A 2024-11-26T16:41:59,924 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:41:59,924 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/C is initiating minor compaction (all files) 2024-11-26T16:41:59,924 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/C in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:59,924 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/5eabfe38d73441a3a29762de22b5c9a7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/aad9b71bb1654bf591b993960311dc2b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/3498596f87d14418a29e2fe21d9787e9] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=36.0 K 2024-11-26T16:41:59,925 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 5eabfe38d73441a3a29762de22b5c9a7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732639315717 2024-11-26T16:41:59,925 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting aad9b71bb1654bf591b993960311dc2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732639316817 2024-11-26T16:41:59,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:59,926 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 3498596f87d14418a29e2fe21d9787e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732639319010 2024-11-26T16:41:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639379917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:59,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639379919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:59,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:41:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639379921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:41:59,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:59,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:41:59,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:41:59,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/3881af98b30a4e23b001f8ff97884df4 is 50, key is test_row_0/B:col10/1732639319207/Put/seqid=0 2024-11-26T16:41:59,942 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#C#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:41:59,943 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/00815924c3e44050a60468afa49919f6 is 50, key is test_row_0/C:col10/1732639319016/Put/seqid=0 2024-11-26T16:41:59,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742105_1281 (size=12151) 2024-11-26T16:41:59,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/3881af98b30a4e23b001f8ff97884df4 2024-11-26T16:41:59,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742106_1282 (size=12663) 2024-11-26T16:41:59,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T16:41:59,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:59,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:41:59,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:41:59,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:59,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:59,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:41:59,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/f95653e7c01d4ac19b3a83258e582ecd is 50, key is test_row_0/C:col10/1732639319207/Put/seqid=0 2024-11-26T16:41:59,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742107_1283 (size=12151) 2024-11-26T16:42:00,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639380031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:00,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639380031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:00,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639380032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:00,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:00,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639380032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:00,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T16:42:00,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:00,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:00,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:00,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:00,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:00,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T16:42:00,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:00,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639380241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:00,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:00,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639380241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:00,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:00,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:00,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639380241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:00,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:00,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:00,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639380241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:00,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:00,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T16:42:00,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:00,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:00,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:00,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:00,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:00,355 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/00815924c3e44050a60468afa49919f6 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/00815924c3e44050a60468afa49919f6 2024-11-26T16:42:00,360 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/C of 8c8b9347f011e2219af1b66f205e204f into 00815924c3e44050a60468afa49919f6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:00,360 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:00,360 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/C, priority=13, startTime=1732639319876; duration=0sec 2024-11-26T16:42:00,360 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:00,360 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:C 2024-11-26T16:42:00,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/f95653e7c01d4ac19b3a83258e582ecd 2024-11-26T16:42:00,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/4a229a99a2e940e6aeb5547af437c9cc as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/4a229a99a2e940e6aeb5547af437c9cc 2024-11-26T16:42:00,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639380371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:00,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:00,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/4a229a99a2e940e6aeb5547af437c9cc, entries=150, sequenceid=237, filesize=11.9 K 2024-11-26T16:42:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/3881af98b30a4e23b001f8ff97884df4 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/3881af98b30a4e23b001f8ff97884df4 2024-11-26T16:42:00,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/3881af98b30a4e23b001f8ff97884df4, entries=150, sequenceid=237, filesize=11.9 K 2024-11-26T16:42:00,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/f95653e7c01d4ac19b3a83258e582ecd as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/f95653e7c01d4ac19b3a83258e582ecd 2024-11-26T16:42:00,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/f95653e7c01d4ac19b3a83258e582ecd, entries=150, sequenceid=237, filesize=11.9 K 2024-11-26T16:42:00,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 8c8b9347f011e2219af1b66f205e204f in 489ms, sequenceid=237, compaction requested=false 2024-11-26T16:42:00,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:00,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-26T16:42:00,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:00,414 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-26T16:42:00,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:42:00,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:00,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:42:00,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:00,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:42:00,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:00,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/ae2485b8f23740a3b445fab1dac305a5 is 50, key is test_row_0/A:col10/1732639319918/Put/seqid=0 2024-11-26T16:42:00,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742108_1284 (size=12151) 2024-11-26T16:42:00,424 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/ae2485b8f23740a3b445fab1dac305a5 2024-11-26T16:42:00,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/b6da699de4f545f6a8dd1ff8ac41022a is 50, key is test_row_0/B:col10/1732639319918/Put/seqid=0 2024-11-26T16:42:00,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742109_1285 (size=12151) 2024-11-26T16:42:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:42:00,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:00,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732639320574, max timestamp=1732639320585Num uniques 859; 2024-11-26T16:42:00,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:42:00,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732639320574, max timestamp=1732639320585Num uniques 859; 2024-11-26T16:42:00,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732639320574, max timestamp=1732639320585Num uniques 859; 2024-11-26T16:42:00,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:42:00,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:42:00,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639380621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:00,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639380622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:00,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:00,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732639320596, max timestamp=1732639320626Num uniques 1068; 2024-11-26T16:42:00,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-26T16:42:00,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732639320596, max timestamp=1732639320626Num uniques 1068; 2024-11-26T16:42:00,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-26T16:42:00,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732639320596, max timestamp=1732639320626Num uniques 1068; 2024-11-26T16:42:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:00,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-26T16:42:00,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639380683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:00,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639380691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:00,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:00,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639380741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:00,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639380742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:00,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:00,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:00,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639380801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:00,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639380801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:00,837 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/b6da699de4f545f6a8dd1ff8ac41022a 2024-11-26T16:42:00,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/058baad17f0e4d329a774c51e34f6348 is 50, key is test_row_0/C:col10/1732639319918/Put/seqid=0 2024-11-26T16:42:00,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742110_1286 (size=12151) 2024-11-26T16:42:00,857 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/058baad17f0e4d329a774c51e34f6348 2024-11-26T16:42:00,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/ae2485b8f23740a3b445fab1dac305a5 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/ae2485b8f23740a3b445fab1dac305a5 2024-11-26T16:42:00,869 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/ae2485b8f23740a3b445fab1dac305a5, entries=150, sequenceid=249, filesize=11.9 K 2024-11-26T16:42:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/b6da699de4f545f6a8dd1ff8ac41022a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/b6da699de4f545f6a8dd1ff8ac41022a 2024-11-26T16:42:00,875 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/b6da699de4f545f6a8dd1ff8ac41022a, entries=150, sequenceid=249, filesize=11.9 K 2024-11-26T16:42:00,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/058baad17f0e4d329a774c51e34f6348 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/058baad17f0e4d329a774c51e34f6348 2024-11-26T16:42:00,879 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/058baad17f0e4d329a774c51e34f6348, entries=150, sequenceid=249, filesize=11.9 K 2024-11-26T16:42:00,880 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 8c8b9347f011e2219af1b66f205e204f in 466ms, sequenceid=249, compaction requested=true 2024-11-26T16:42:00,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:00,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:00,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-26T16:42:00,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-26T16:42:00,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-26T16:42:00,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8470 sec 2024-11-26T16:42:00,886 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.8530 sec 2024-11-26T16:42:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:42:00,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-11-26T16:42:00,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:42:00,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:42:00,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:42:00,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:42:00,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:42:00,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:42:00,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/0eb03a2e91904d57a9fb63511af080fc is 50, key is test_row_0/A:col10/1732639320955/Put/seqid=0 2024-11-26T16:42:00,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:00,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639380970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:00,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742111_1287 (size=14741) 2024-11-26T16:42:00,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:00,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/0eb03a2e91904d57a9fb63511af080fc 2024-11-26T16:42:00,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639380976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:00,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/5d98f077f0b145dea4630763287b7141 is 50, key is test_row_0/B:col10/1732639320955/Put/seqid=0 2024-11-26T16:42:00,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742112_1288 (size=12301) 2024-11-26T16:42:00,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/5d98f077f0b145dea4630763287b7141 2024-11-26T16:42:01,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/2f005e49ecdc4421960f71561fdd3dcf is 50, key is test_row_0/C:col10/1732639320955/Put/seqid=0 2024-11-26T16:42:01,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742113_1289 (size=12301) 2024-11-26T16:42:01,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639381011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639381011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:01,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:01,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:01,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639381081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:01,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:01,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639381091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:01,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-26T16:42:01,186 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:01,187 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:42:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-26T16:42:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T16:42:01,189 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:42:01,189 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:42:01,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:42:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T16:42:01,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639381290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:01,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:01,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639381301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:01,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:01,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639381321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:01,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639381321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:01,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T16:42:01,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:01,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:01,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:01,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:01,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:01,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639381381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:01,387 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in 8c8b9347f011e2219af1b66f205e204f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:01,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/2f005e49ecdc4421960f71561fdd3dcf 2024-11-26T16:42:01,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/0eb03a2e91904d57a9fb63511af080fc as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/0eb03a2e91904d57a9fb63511af080fc 2024-11-26T16:42:01,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/0eb03a2e91904d57a9fb63511af080fc, entries=200, sequenceid=304, filesize=14.4 K 2024-11-26T16:42:01,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/5d98f077f0b145dea4630763287b7141 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/5d98f077f0b145dea4630763287b7141 2024-11-26T16:42:01,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/5d98f077f0b145dea4630763287b7141, entries=150, sequenceid=304, filesize=12.0 K 2024-11-26T16:42:01,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/2f005e49ecdc4421960f71561fdd3dcf as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/2f005e49ecdc4421960f71561fdd3dcf 2024-11-26T16:42:01,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/2f005e49ecdc4421960f71561fdd3dcf, entries=150, sequenceid=304, filesize=12.0 K 2024-11-26T16:42:01,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for 8c8b9347f011e2219af1b66f205e204f in 472ms, sequenceid=304, compaction requested=true 2024-11-26T16:42:01,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:01,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:42:01,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:01,429 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:01,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:42:01,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:01,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:01,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:01,429 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:01,431 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:01,431 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51706 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:01,431 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/B is initiating minor compaction (all files) 2024-11-26T16:42:01,431 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/A is initiating minor compaction (all files) 2024-11-26T16:42:01,431 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/A in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:01,431 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/B in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:01,431 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/5cbd650499a9488ba8a79f763906e620, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/4a229a99a2e940e6aeb5547af437c9cc, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/ae2485b8f23740a3b445fab1dac305a5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/0eb03a2e91904d57a9fb63511af080fc] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=50.5 K 2024-11-26T16:42:01,431 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/0aeee3db914744939f129e98fe85803a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/3881af98b30a4e23b001f8ff97884df4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/b6da699de4f545f6a8dd1ff8ac41022a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/5d98f077f0b145dea4630763287b7141] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=48.1 K 2024-11-26T16:42:01,432 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aeee3db914744939f129e98fe85803a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732639319010 2024-11-26T16:42:01,432 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cbd650499a9488ba8a79f763906e620, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732639319010 2024-11-26T16:42:01,432 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a229a99a2e940e6aeb5547af437c9cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732639319193 2024-11-26T16:42:01,432 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 3881af98b30a4e23b001f8ff97884df4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732639319193 2024-11-26T16:42:01,433 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae2485b8f23740a3b445fab1dac305a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732639319905 2024-11-26T16:42:01,433 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting b6da699de4f545f6a8dd1ff8ac41022a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732639319905 2024-11-26T16:42:01,433 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eb03a2e91904d57a9fb63511af080fc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732639320664 2024-11-26T16:42:01,433 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d98f077f0b145dea4630763287b7141, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732639320664 2024-11-26T16:42:01,446 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#A#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:01,447 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#B#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:01,447 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/71e7e6ebc8354f0a9e2521f8319a3bd7 is 50, key is test_row_0/A:col10/1732639320955/Put/seqid=0 2024-11-26T16:42:01,447 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/49fcfdf11b7c42ad973e15475466c5ad is 50, key is test_row_0/B:col10/1732639320955/Put/seqid=0 2024-11-26T16:42:01,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742115_1291 (size=12949) 2024-11-26T16:42:01,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742114_1290 (size=12949) 2024-11-26T16:42:01,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-26T16:42:01,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:01,497 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-26T16:42:01,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:42:01,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:01,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:42:01,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:01,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:42:01,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T16:42:01,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/a9c09d412de44839a482f2a7d0c95db7 is 50, key is test_row_0/A:col10/1732639320966/Put/seqid=0 2024-11-26T16:42:01,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742116_1292 (size=12301) 2024-11-26T16:42:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:42:01,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:01,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639321640, max timestamp=1732639321655Num uniques 859; 2024-11-26T16:42:01,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639321640, max timestamp=1732639321655Num uniques 859; 2024-11-26T16:42:01,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:01,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:01,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732639321640, max timestamp=1732639321655Num uniques 859; 2024-11-26T16:42:01,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-26T16:42:01,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732639321690, max timestamp=1732639321715Num uniques 1018; 2024-11-26T16:42:01,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T16:42:01,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732639321690, max timestamp=1732639321715Num uniques 1018; 2024-11-26T16:42:01,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T16:42:01,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732639321690, max timestamp=1732639321715Num uniques 1009; 2024-11-26T16:42:01,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-26T16:42:01,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732639321765, max timestamp=1732639321779Num uniques 1027; 2024-11-26T16:42:01,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-26T16:42:01,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732639321765, max timestamp=1732639321779Num uniques 1027; 2024-11-26T16:42:01,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-26T16:42:01,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732639321765, max timestamp=1732639321779Num uniques 1027; 2024-11-26T16:42:01,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-26T16:42:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T16:42:01,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:01,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639381839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:01,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639381840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:01,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639381841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:01,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732639321837, max timestamp=1732639321848Num uniques 1086; 2024-11-26T16:42:01,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-26T16:42:01,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732639321837, max timestamp=1732639321848Num uniques 1086; 2024-11-26T16:42:01,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-26T16:42:01,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732639321837, max timestamp=1732639321848Num uniques 1086; 2024-11-26T16:42:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-26T16:42:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:01,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:01,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:01,874 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/71e7e6ebc8354f0a9e2521f8319a3bd7 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/71e7e6ebc8354f0a9e2521f8319a3bd7 2024-11-26T16:42:01,874 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/49fcfdf11b7c42ad973e15475466c5ad as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/49fcfdf11b7c42ad973e15475466c5ad 2024-11-26T16:42:01,880 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/A of 8c8b9347f011e2219af1b66f205e204f into 71e7e6ebc8354f0a9e2521f8319a3bd7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:01,880 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:01,880 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/A, priority=12, startTime=1732639321429; duration=0sec 2024-11-26T16:42:01,880 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:01,880 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:A 2024-11-26T16:42:01,880 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:01,880 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/B of 8c8b9347f011e2219af1b66f205e204f into 49fcfdf11b7c42ad973e15475466c5ad(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:01,880 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:01,881 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/B, priority=12, startTime=1732639321429; duration=0sec 2024-11-26T16:42:01,881 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:01,881 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:B 2024-11-26T16:42:01,881 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:01,882 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/C is initiating minor compaction (all files) 2024-11-26T16:42:01,882 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/C in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:01,882 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/00815924c3e44050a60468afa49919f6, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/f95653e7c01d4ac19b3a83258e582ecd, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/058baad17f0e4d329a774c51e34f6348, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/2f005e49ecdc4421960f71561fdd3dcf] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=48.1 K 2024-11-26T16:42:01,882 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00815924c3e44050a60468afa49919f6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732639319010 2024-11-26T16:42:01,883 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting f95653e7c01d4ac19b3a83258e582ecd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732639319193 2024-11-26T16:42:01,884 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 058baad17f0e4d329a774c51e34f6348, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732639319905 2024-11-26T16:42:01,884 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f005e49ecdc4421960f71561fdd3dcf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732639320664 2024-11-26T16:42:01,893 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#C#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:01,894 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/ba92045f39294d89b84375cb6bcec489 is 50, key is test_row_0/C:col10/1732639320955/Put/seqid=0 2024-11-26T16:42:01,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742117_1293 (size=12949) 2024-11-26T16:42:01,907 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/a9c09d412de44839a482f2a7d0c95db7 2024-11-26T16:42:01,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/a66b0623db3343c6a12c53cea71bf490 is 50, key is test_row_0/B:col10/1732639320966/Put/seqid=0 2024-11-26T16:42:01,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742118_1294 (size=12301) 2024-11-26T16:42:01,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639381964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:01,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732639321940, max timestamp=1732639321966Num uniques 1045; 2024-11-26T16:42:01,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732639321940, max timestamp=1732639321966Num uniques 1045; 2024-11-26T16:42:01,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-26T16:42:01,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732639321940, max timestamp=1732639321966Num uniques 1045; 2024-11-26T16:42:01,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-26T16:42:01,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:01,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:01,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-26T16:42:02,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639382006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:02,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:02,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:02,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639382008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:02,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:02,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:02,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639382009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:02,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:02,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639382081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:02,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639382121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:02,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639382121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:02,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639382121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:02,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:02,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639382291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:02,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:02,308 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/ba92045f39294d89b84375cb6bcec489 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/ba92045f39294d89b84375cb6bcec489 2024-11-26T16:42:02,312 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/C of 8c8b9347f011e2219af1b66f205e204f into ba92045f39294d89b84375cb6bcec489(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:02,313 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:02,313 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/C, priority=12, startTime=1732639321429; duration=0sec 2024-11-26T16:42:02,313 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:02,313 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:C 2024-11-26T16:42:02,318 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/a66b0623db3343c6a12c53cea71bf490 2024-11-26T16:42:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T16:42:02,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/e4692c19ad51401488acd8a5b39e9ea7 is 50, key is test_row_0/C:col10/1732639320966/Put/seqid=0 2024-11-26T16:42:02,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742119_1295 (size=12301) 2024-11-26T16:42:02,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639382331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:02,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:02,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639382331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:02,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639382331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:02,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:02,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:02,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639382601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:02,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:02,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639382642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:02,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:02,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639382642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639382642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:02,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:02,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:02,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:02,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:02,733 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/e4692c19ad51401488acd8a5b39e9ea7 2024-11-26T16:42:02,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/a9c09d412de44839a482f2a7d0c95db7 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/a9c09d412de44839a482f2a7d0c95db7 2024-11-26T16:42:02,743 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/a9c09d412de44839a482f2a7d0c95db7, entries=150, sequenceid=312, filesize=12.0 K 2024-11-26T16:42:02,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/a66b0623db3343c6a12c53cea71bf490 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a66b0623db3343c6a12c53cea71bf490 2024-11-26T16:42:02,748 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a66b0623db3343c6a12c53cea71bf490, entries=150, sequenceid=312, filesize=12.0 K 2024-11-26T16:42:02,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/e4692c19ad51401488acd8a5b39e9ea7 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/e4692c19ad51401488acd8a5b39e9ea7 2024-11-26T16:42:02,752 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/e4692c19ad51401488acd8a5b39e9ea7, entries=150, sequenceid=312, filesize=12.0 K 2024-11-26T16:42:02,753 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 8c8b9347f011e2219af1b66f205e204f in 1256ms, sequenceid=312, compaction requested=false 2024-11-26T16:42:02,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:02,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:02,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-26T16:42:02,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-26T16:42:02,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-26T16:42:02,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5650 sec 2024-11-26T16:42:02,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.5700 sec 2024-11-26T16:42:03,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=154.31 KB heapSize=432 KB 2024-11-26T16:42:03,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:42:03,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:03,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:42:03,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:03,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:42:03,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:42:03,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/d40986d1d7444617836b29f21524a0ac is 50, key is test_row_0/A:col10/1732639321999/Put/seqid=0 2024-11-26T16:42:03,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742120_1296 (size=12301) 2024-11-26T16:42:03,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639383149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:03,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:03,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:03,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:03,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639383152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:03,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639383151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:03,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639383161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:03,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639383261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-26T16:42:03,331 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:03,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:42:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-26T16:42:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T16:42:03,333 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:42:03,334 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:42:03,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:42:03,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639383401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:03,405 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in 8c8b9347f011e2219af1b66f205e204f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:03,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:03,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:03,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T16:42:03,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639383471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:03,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:03,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:03,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:03,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T16:42:03,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:03,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:03,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:03,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:03,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:03,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/d40986d1d7444617836b29f21524a0ac 2024-11-26T16:42:03,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/41cd4541d91f4601ac37e858f4651a8a is 50, key is test_row_0/B:col10/1732639321999/Put/seqid=0 2024-11-26T16:42:03,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742121_1297 (size=12301) 2024-11-26T16:42:03,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T16:42:03,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:03,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:03,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:03,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:03,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T16:42:03,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639383781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:03,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:03,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T16:42:03,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:03,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:03,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:03,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:03,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:03,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T16:42:03,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:03,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:03,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:03,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:03,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:03,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/41cd4541d91f4601ac37e858f4651a8a 2024-11-26T16:42:03,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/6803605bd57e454a8a3b4e1baac17439 is 50, key is test_row_0/C:col10/1732639321999/Put/seqid=0 2024-11-26T16:42:03,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T16:42:03,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742122_1298 (size=12301) 2024-11-26T16:42:04,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T16:42:04,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:04,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:04,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:04,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:04,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:04,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639384171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:04,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639384171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:04,176 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_0' in 8c8b9347f011e2219af1b66f205e204f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:04,176 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in 8c8b9347f011e2219af1b66f205e204f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:04,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:04,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639384181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:04,184 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in 8c8b9347f011e2219af1b66f205e204f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:04,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T16:42:04,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:04,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:04,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:04,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:04,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:04,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639384291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:04,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/6803605bd57e454a8a3b4e1baac17439 2024-11-26T16:42:04,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/d40986d1d7444617836b29f21524a0ac as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/d40986d1d7444617836b29f21524a0ac 2024-11-26T16:42:04,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/d40986d1d7444617836b29f21524a0ac, entries=150, sequenceid=421, filesize=12.0 K 2024-11-26T16:42:04,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/41cd4541d91f4601ac37e858f4651a8a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/41cd4541d91f4601ac37e858f4651a8a 2024-11-26T16:42:04,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/41cd4541d91f4601ac37e858f4651a8a, entries=150, sequenceid=421, filesize=12.0 K 2024-11-26T16:42:04,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/6803605bd57e454a8a3b4e1baac17439 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/6803605bd57e454a8a3b4e1baac17439 2024-11-26T16:42:04,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/6803605bd57e454a8a3b4e1baac17439, entries=150, sequenceid=421, filesize=12.0 K 2024-11-26T16:42:04,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~431.95 KB/442320, currentSize=26.84 KB/27480 for 8c8b9347f011e2219af1b66f205e204f in 1275ms, sequenceid=421, compaction requested=true 2024-11-26T16:42:04,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:04,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:42:04,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:04,391 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:04,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:42:04,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:04,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:04,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:04,391 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:04,392 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:04,392 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:04,393 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/A is initiating minor compaction (all files) 2024-11-26T16:42:04,393 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/B is initiating minor compaction (all files) 2024-11-26T16:42:04,393 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/A in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:04,393 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/B in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:04,393 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/71e7e6ebc8354f0a9e2521f8319a3bd7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/a9c09d412de44839a482f2a7d0c95db7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/d40986d1d7444617836b29f21524a0ac] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=36.7 K 2024-11-26T16:42:04,393 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/49fcfdf11b7c42ad973e15475466c5ad, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a66b0623db3343c6a12c53cea71bf490, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/41cd4541d91f4601ac37e858f4651a8a] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=36.7 K 2024-11-26T16:42:04,393 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71e7e6ebc8354f0a9e2521f8319a3bd7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732639320664 2024-11-26T16:42:04,393 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 49fcfdf11b7c42ad973e15475466c5ad, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732639320664 2024-11-26T16:42:04,394 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9c09d412de44839a482f2a7d0c95db7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732639320966 2024-11-26T16:42:04,394 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a66b0623db3343c6a12c53cea71bf490, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732639320966 2024-11-26T16:42:04,397 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting d40986d1d7444617836b29f21524a0ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732639321999 2024-11-26T16:42:04,397 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 41cd4541d91f4601ac37e858f4651a8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732639321999 2024-11-26T16:42:04,408 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#A#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:04,409 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/f78365b080db44669a251200843e05ba is 50, key is test_row_0/A:col10/1732639321999/Put/seqid=0 2024-11-26T16:42:04,412 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#B#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:04,412 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/a52bec8434394c089bcb9e00a5c631b7 is 50, key is test_row_0/B:col10/1732639321999/Put/seqid=0 2024-11-26T16:42:04,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-26T16:42:04,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:04,415 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-26T16:42:04,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:42:04,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:04,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:42:04,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:04,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:42:04,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:04,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742123_1299 (size=13051) 2024-11-26T16:42:04,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/c9a329e3ff134f98915b65012b715499 is 50, key is test_row_1/A:col10/1732639323129/Put/seqid=0 2024-11-26T16:42:04,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742124_1300 (size=13051) 2024-11-26T16:42:04,430 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/f78365b080db44669a251200843e05ba as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/f78365b080db44669a251200843e05ba 2024-11-26T16:42:04,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742125_1301 (size=9857) 2024-11-26T16:42:04,439 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/A of 8c8b9347f011e2219af1b66f205e204f into f78365b080db44669a251200843e05ba(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:04,439 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:04,439 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/c9a329e3ff134f98915b65012b715499 2024-11-26T16:42:04,439 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/A, priority=13, startTime=1732639324391; duration=0sec 2024-11-26T16:42:04,439 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:04,439 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:A 2024-11-26T16:42:04,439 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:04,440 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:04,441 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/C is initiating minor compaction (all files) 2024-11-26T16:42:04,441 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/C in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:04,441 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/ba92045f39294d89b84375cb6bcec489, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/e4692c19ad51401488acd8a5b39e9ea7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/6803605bd57e454a8a3b4e1baac17439] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=36.7 K 2024-11-26T16:42:04,441 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba92045f39294d89b84375cb6bcec489, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732639320664 2024-11-26T16:42:04,442 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4692c19ad51401488acd8a5b39e9ea7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732639320966 2024-11-26T16:42:04,442 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6803605bd57e454a8a3b4e1baac17439, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732639321999 2024-11-26T16:42:04,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/d3a499e4d1e54b8f9dbf6b49f6f7592e is 50, key is test_row_1/B:col10/1732639323129/Put/seqid=0 2024-11-26T16:42:04,456 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#C#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:04,457 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/71a1ef0d2f9143b483d0ca4a2ea90dc9 is 50, key is test_row_0/C:col10/1732639321999/Put/seqid=0 2024-11-26T16:42:04,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742126_1302 (size=9857) 2024-11-26T16:42:04,460 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/d3a499e4d1e54b8f9dbf6b49f6f7592e 2024-11-26T16:42:04,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742127_1303 (size=13051) 2024-11-26T16:42:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T16:42:04,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/9cad54294c0b4e53b2706c6cd142d343 is 50, key is test_row_1/C:col10/1732639323129/Put/seqid=0 2024-11-26T16:42:04,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742128_1304 (size=9857) 2024-11-26T16:42:04,482 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/9cad54294c0b4e53b2706c6cd142d343 2024-11-26T16:42:04,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/c9a329e3ff134f98915b65012b715499 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/c9a329e3ff134f98915b65012b715499 2024-11-26T16:42:04,493 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/c9a329e3ff134f98915b65012b715499, entries=100, sequenceid=428, filesize=9.6 K 2024-11-26T16:42:04,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/d3a499e4d1e54b8f9dbf6b49f6f7592e as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/d3a499e4d1e54b8f9dbf6b49f6f7592e 2024-11-26T16:42:04,498 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/d3a499e4d1e54b8f9dbf6b49f6f7592e, entries=100, sequenceid=428, filesize=9.6 K 2024-11-26T16:42:04,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/9cad54294c0b4e53b2706c6cd142d343 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/9cad54294c0b4e53b2706c6cd142d343 2024-11-26T16:42:04,503 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/9cad54294c0b4e53b2706c6cd142d343, entries=100, sequenceid=428, filesize=9.6 K 2024-11-26T16:42:04,504 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 8c8b9347f011e2219af1b66f205e204f in 89ms, sequenceid=428, compaction requested=false 2024-11-26T16:42:04,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:04,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:04,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-26T16:42:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-26T16:42:04,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-26T16:42:04,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1720 sec 2024-11-26T16:42:04,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.1780 sec 2024-11-26T16:42:04,835 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/a52bec8434394c089bcb9e00a5c631b7 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a52bec8434394c089bcb9e00a5c631b7 2024-11-26T16:42:04,841 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/B of 8c8b9347f011e2219af1b66f205e204f into a52bec8434394c089bcb9e00a5c631b7(size=12.7 K), total size for store is 22.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:04,841 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:04,841 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/B, priority=13, startTime=1732639324391; duration=0sec 2024-11-26T16:42:04,841 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:04,841 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:B 2024-11-26T16:42:04,881 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/71a1ef0d2f9143b483d0ca4a2ea90dc9 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/71a1ef0d2f9143b483d0ca4a2ea90dc9 2024-11-26T16:42:04,887 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/C of 8c8b9347f011e2219af1b66f205e204f into 71a1ef0d2f9143b483d0ca4a2ea90dc9(size=12.7 K), total size for store is 22.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:04,887 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:04,888 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/C, priority=13, startTime=1732639324391; duration=0sec 2024-11-26T16:42:04,888 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:04,888 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:C 2024-11-26T16:42:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:42:05,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T16:42:05,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:42:05,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:05,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:42:05,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:05,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:42:05,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:05,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/cc59d30f8d494f7f94fae642c51eecd2 is 50, key is test_row_0/A:col10/1732639325308/Put/seqid=0 2024-11-26T16:42:05,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742129_1305 (size=12301) 2024-11-26T16:42:05,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639325383, max timestamp=1732639325406Num uniques 859; 2024-11-26T16:42:05,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639325383, max timestamp=1732639325406Num uniques 859; 2024-11-26T16:42:05,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639325383, max timestamp=1732639325406Num uniques 859; 2024-11-26T16:42:05,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:05,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:05,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-26T16:42:05,481 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:05,482 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:42:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-26T16:42:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T16:42:05,483 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:42:05,484 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:42:05,484 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:42:05,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639385501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T16:42:05,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639385611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:05,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:05,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:05,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T16:42:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:05,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:05,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/cc59d30f8d494f7f94fae642c51eecd2 2024-11-26T16:42:05,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/3c4b6b774773474881335cd6ad1de12a is 50, key is test_row_0/B:col10/1732639325308/Put/seqid=0 2024-11-26T16:42:05,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742130_1306 (size=12301) 2024-11-26T16:42:05,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T16:42:05,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:05,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:05,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:05,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:05,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T16:42:05,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:05,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639385821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:05,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:05,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:05,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:05,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T16:42:05,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:05,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:05,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:05,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:05,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:06,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T16:42:06,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:06,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:06,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:06,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:06,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T16:42:06,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639386131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:06,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/3c4b6b774773474881335cd6ad1de12a 2024-11-26T16:42:06,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/784abc55f08a464381b589c044b3cdd0 is 50, key is test_row_0/C:col10/1732639325308/Put/seqid=0 2024-11-26T16:42:06,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742131_1307 (size=12301) 2024-11-26T16:42:06,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/784abc55f08a464381b589c044b3cdd0 2024-11-26T16:42:06,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/cc59d30f8d494f7f94fae642c51eecd2 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/cc59d30f8d494f7f94fae642c51eecd2 2024-11-26T16:42:06,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/cc59d30f8d494f7f94fae642c51eecd2, entries=150, sequenceid=441, filesize=12.0 K 2024-11-26T16:42:06,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/3c4b6b774773474881335cd6ad1de12a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/3c4b6b774773474881335cd6ad1de12a 2024-11-26T16:42:06,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/3c4b6b774773474881335cd6ad1de12a, entries=150, sequenceid=441, filesize=12.0 K 2024-11-26T16:42:06,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/784abc55f08a464381b589c044b3cdd0 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/784abc55f08a464381b589c044b3cdd0 2024-11-26T16:42:06,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/784abc55f08a464381b589c044b3cdd0, entries=150, sequenceid=441, filesize=12.0 K 2024-11-26T16:42:06,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 8c8b9347f011e2219af1b66f205e204f in 857ms, sequenceid=441, compaction requested=true 2024-11-26T16:42:06,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:06,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:42:06,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:06,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:42:06,174 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:06,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:06,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:06,174 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:06,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:06,174 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:06,175 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:06,175 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/A is initiating minor compaction (all files) 2024-11-26T16:42:06,175 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/B is initiating minor compaction (all files) 2024-11-26T16:42:06,175 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/A in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:06,175 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/B in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:06,175 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/f78365b080db44669a251200843e05ba, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/c9a329e3ff134f98915b65012b715499, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/cc59d30f8d494f7f94fae642c51eecd2] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=34.4 K 2024-11-26T16:42:06,175 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a52bec8434394c089bcb9e00a5c631b7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/d3a499e4d1e54b8f9dbf6b49f6f7592e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/3c4b6b774773474881335cd6ad1de12a] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=34.4 K 2024-11-26T16:42:06,175 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a52bec8434394c089bcb9e00a5c631b7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732639321999 2024-11-26T16:42:06,175 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting f78365b080db44669a251200843e05ba, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732639321999 2024-11-26T16:42:06,176 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting d3a499e4d1e54b8f9dbf6b49f6f7592e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732639323129 2024-11-26T16:42:06,176 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9a329e3ff134f98915b65012b715499, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732639323129 2024-11-26T16:42:06,176 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c4b6b774773474881335cd6ad1de12a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732639325308 2024-11-26T16:42:06,176 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc59d30f8d494f7f94fae642c51eecd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732639325308 2024-11-26T16:42:06,183 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#B#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:06,184 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/a748304b68eb43a3bd76f4b4087fa71b is 50, key is test_row_0/B:col10/1732639325308/Put/seqid=0 2024-11-26T16:42:06,186 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#A#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:06,187 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/a8a500b16d4f43fabb7951af4c814a0b is 50, key is test_row_0/A:col10/1732639325308/Put/seqid=0 2024-11-26T16:42:06,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742132_1308 (size=13153) 2024-11-26T16:42:06,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742133_1309 (size=13153) 2024-11-26T16:42:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:42:06,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-26T16:42:06,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:42:06,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:42:06,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:42:06,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:42:06,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:42:06,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:42:06,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/874a34dc8ff7480d8f26e6e07491c8e8 is 50, key is test_row_0/A:col10/1732639326194/Put/seqid=0 2024-11-26T16:42:06,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742134_1310 (size=14741) 2024-11-26T16:42:06,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/874a34dc8ff7480d8f26e6e07491c8e8 2024-11-26T16:42:06,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639386209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:06,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:06,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/0c443cf3b82d42119bce9453a9e1a8e1 is 50, key is test_row_0/B:col10/1732639326194/Put/seqid=0 2024-11-26T16:42:06,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742135_1311 (size=12301) 2024-11-26T16:42:06,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639386211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:06,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639386213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:06,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:06,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T16:42:06,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:06,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:06,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:06,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:06,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:06,305 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T16:42:06,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639386320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:06,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:06,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639386321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:06,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:06,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639386321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:06,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T16:42:06,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:06,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:06,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:06,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:06,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:06,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639386531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:06,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639386531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:06,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639386532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:06,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T16:42:06,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:06,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:06,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:06,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:06,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:06,594 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/a748304b68eb43a3bd76f4b4087fa71b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a748304b68eb43a3bd76f4b4087fa71b 2024-11-26T16:42:06,599 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/a8a500b16d4f43fabb7951af4c814a0b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/a8a500b16d4f43fabb7951af4c814a0b 2024-11-26T16:42:06,599 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/B of 8c8b9347f011e2219af1b66f205e204f into a748304b68eb43a3bd76f4b4087fa71b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:06,599 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:06,599 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/B, priority=13, startTime=1732639326174; duration=0sec 2024-11-26T16:42:06,599 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:06,599 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:B 2024-11-26T16:42:06,599 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:06,600 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:06,600 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/C is initiating minor compaction (all files) 2024-11-26T16:42:06,600 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/C in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:06,601 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/71a1ef0d2f9143b483d0ca4a2ea90dc9, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/9cad54294c0b4e53b2706c6cd142d343, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/784abc55f08a464381b589c044b3cdd0] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=34.4 K 2024-11-26T16:42:06,601 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 71a1ef0d2f9143b483d0ca4a2ea90dc9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732639321999 2024-11-26T16:42:06,601 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cad54294c0b4e53b2706c6cd142d343, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732639323129 2024-11-26T16:42:06,602 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 784abc55f08a464381b589c044b3cdd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732639325308 2024-11-26T16:42:06,604 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/A of 8c8b9347f011e2219af1b66f205e204f into a8a500b16d4f43fabb7951af4c814a0b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:06,604 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:06,604 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/A, priority=13, startTime=1732639326173; duration=0sec 2024-11-26T16:42:06,604 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:06,604 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:A 2024-11-26T16:42:06,610 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#C#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:06,610 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/1e9d65282a484941b4f7c7da09357dd6 is 50, key is test_row_0/C:col10/1732639325308/Put/seqid=0 2024-11-26T16:42:06,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742136_1312 (size=13153) 2024-11-26T16:42:06,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/0c443cf3b82d42119bce9453a9e1a8e1 2024-11-26T16:42:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T16:42:06,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/b957db98d2a94a62afd7fc5709723c78 is 50, key is test_row_0/C:col10/1732639326194/Put/seqid=0 2024-11-26T16:42:06,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742137_1313 (size=12301) 2024-11-26T16:42:06,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639386641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:06,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:06,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T16:42:06,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:06,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:06,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:06,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:06,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:06,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639386841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:06,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639386841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:06,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639386842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:06,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T16:42:06,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:06,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:06,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:06,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:06,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:07,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T16:42:07,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:07,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:07,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:07,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:07,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:07,029 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/1e9d65282a484941b4f7c7da09357dd6 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/1e9d65282a484941b4f7c7da09357dd6 2024-11-26T16:42:07,034 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/C of 8c8b9347f011e2219af1b66f205e204f into 1e9d65282a484941b4f7c7da09357dd6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:07,034 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:07,034 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/C, priority=13, startTime=1732639326174; duration=0sec 2024-11-26T16:42:07,034 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:07,034 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:C 2024-11-26T16:42:07,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/b957db98d2a94a62afd7fc5709723c78 2024-11-26T16:42:07,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/874a34dc8ff7480d8f26e6e07491c8e8 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/874a34dc8ff7480d8f26e6e07491c8e8 2024-11-26T16:42:07,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/874a34dc8ff7480d8f26e6e07491c8e8, entries=200, sequenceid=479, filesize=14.4 K 2024-11-26T16:42:07,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/0c443cf3b82d42119bce9453a9e1a8e1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/0c443cf3b82d42119bce9453a9e1a8e1 2024-11-26T16:42:07,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/0c443cf3b82d42119bce9453a9e1a8e1, entries=150, sequenceid=479, filesize=12.0 K 2024-11-26T16:42:07,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/b957db98d2a94a62afd7fc5709723c78 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/b957db98d2a94a62afd7fc5709723c78 2024-11-26T16:42:07,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/b957db98d2a94a62afd7fc5709723c78, entries=150, sequenceid=479, filesize=12.0 K 2024-11-26T16:42:07,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 8c8b9347f011e2219af1b66f205e204f in 864ms, sequenceid=479, compaction requested=false 2024-11-26T16:42:07,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:07,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-26T16:42:07,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:07,173 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-26T16:42:07,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:42:07,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:07,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:42:07,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:07,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:42:07,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:07,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/6fa7ad48d77940f59e89463ce6c2c8a0 is 50, key is test_row_0/A:col10/1732639326210/Put/seqid=0 2024-11-26T16:42:07,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742138_1314 (size=12301) 2024-11-26T16:42:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:42:07,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. as already flushing 2024-11-26T16:42:07,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639327373, max timestamp=1732639327399Num uniques 859; 2024-11-26T16:42:07,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:07,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639327373, max timestamp=1732639327399Num uniques 859; 2024-11-26T16:42:07,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:07,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639327373, max timestamp=1732639327399Num uniques 859; 2024-11-26T16:42:07,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:07,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639387452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:07,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:07,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639387453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:07,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732639327442, max timestamp=1732639327454Num uniques 1018; 2024-11-26T16:42:07,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-26T16:42:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732639327442, max timestamp=1732639327454Num uniques 1018; 2024-11-26T16:42:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:07,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-26T16:42:07,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732639327442, max timestamp=1732639327454Num uniques 1018; 2024-11-26T16:42:07,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-26T16:42:07,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639387518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:07,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639387519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:07,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:07,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639387571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:07,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:07,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:07,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639387571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:07,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:07,588 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/6fa7ad48d77940f59e89463ce6c2c8a0 2024-11-26T16:42:07,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/c30a19c0f3844a71923543b79e898674 is 50, key is test_row_0/B:col10/1732639326210/Put/seqid=0 2024-11-26T16:42:07,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742139_1315 (size=12301) 2024-11-26T16:42:07,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:07,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639387631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:07,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639387631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T16:42:07,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53292 deadline: 1732639387660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:07,665 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_0' in 8c8b9347f011e2219af1b66f205e204f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:07,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:07,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:07,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639387781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:07,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:07,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639387781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:07,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:07,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:07,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:07,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639387841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:07,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639387841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:08,002 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/c30a19c0f3844a71923543b79e898674 2024-11-26T16:42:08,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/f0d755a8b80245008600434bb69df9d7 is 50, key is test_row_0/C:col10/1732639326210/Put/seqid=0 2024-11-26T16:42:08,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742140_1316 (size=12301) 2024-11-26T16:42:08,014 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/f0d755a8b80245008600434bb69df9d7 2024-11-26T16:42:08,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/6fa7ad48d77940f59e89463ce6c2c8a0 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/6fa7ad48d77940f59e89463ce6c2c8a0 2024-11-26T16:42:08,023 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/6fa7ad48d77940f59e89463ce6c2c8a0, entries=150, sequenceid=491, filesize=12.0 K 2024-11-26T16:42:08,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/c30a19c0f3844a71923543b79e898674 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/c30a19c0f3844a71923543b79e898674 2024-11-26T16:42:08,029 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/c30a19c0f3844a71923543b79e898674, entries=150, sequenceid=491, filesize=12.0 K 2024-11-26T16:42:08,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/f0d755a8b80245008600434bb69df9d7 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/f0d755a8b80245008600434bb69df9d7 2024-11-26T16:42:08,034 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/f0d755a8b80245008600434bb69df9d7, entries=150, sequenceid=491, filesize=12.0 K 2024-11-26T16:42:08,035 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for 8c8b9347f011e2219af1b66f205e204f in 862ms, sequenceid=491, compaction requested=true 2024-11-26T16:42:08,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:08,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:08,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-26T16:42:08,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-26T16:42:08,039 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-26T16:42:08,039 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5530 sec 2024-11-26T16:42:08,041 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.5580 sec 2024-11-26T16:42:08,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:42:08,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=154.31 KB heapSize=435.23 KB 2024-11-26T16:42:08,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:42:08,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:42:08,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:42:08,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:42:08,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:42:08,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:42:08,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/96927b4d56e94d529db5a35652f43aad is 50, key is test_row_0/A:col10/1732639328094/Put/seqid=0 2024-11-26T16:42:08,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742141_1317 (size=14741) 2024-11-26T16:42:08,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639388106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:08,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639388112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:08,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:08,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:08,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639388151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:08,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:08,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639388151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:08,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:08,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:08,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:08,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639388221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:08,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639388221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:08,423 INFO [Thread-1098 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1098. 2024-11-26T16:42:08,423 DEBUG [Thread-1098 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:42:08,423 DEBUG [Thread-1098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:08,423 DEBUG [Thread-1098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:08,423 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:08,424 INFO [Thread-1102 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1102. 2024-11-26T16:42:08,425 DEBUG [Thread-1102 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:42:08,425 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:08,425 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:08,425 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:08,428 INFO [Thread-1100 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1100. 2024-11-26T16:42:08,428 DEBUG [Thread-1100 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:42:08,428 DEBUG [Thread-1100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:08,428 DEBUG [Thread-1100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:08,428 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:08,429 INFO [Thread-1106 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1106. 2024-11-26T16:42:08,429 DEBUG [Thread-1106 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:42:08,429 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:08,429 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:08,429 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:08,429 INFO [Thread-1104 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1104. 2024-11-26T16:42:08,429 DEBUG [Thread-1104 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:42:08,429 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:08,429 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:08,429 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:08,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:08,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:08,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639388431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:08,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639388431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:08,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:08,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/96927b4d56e94d529db5a35652f43aad 2024-11-26T16:42:08,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/8fcd699ce86941b786b7ff5b5e8c5299 is 50, key is test_row_0/B:col10/1732639328094/Put/seqid=0 2024-11-26T16:42:08,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742142_1318 (size=12301) 2024-11-26T16:42:08,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:08,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:08,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53272 deadline: 1732639388661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:08,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53264 deadline: 1732639388661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:08,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:08,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:08,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:08,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:08,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639388741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:08,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639388741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:08,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/8fcd699ce86941b786b7ff5b5e8c5299 2024-11-26T16:42:08,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/f1b968f67b134184a7853f71c7161f90 is 50, key is test_row_0/C:col10/1732639328094/Put/seqid=0 2024-11-26T16:42:08,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742143_1319 (size=12301) 2024-11-26T16:42:09,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:09,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53244 deadline: 1732639389251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53286 deadline: 1732639389251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:09,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:09,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:09,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:09,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:09,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:09,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:09,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/f1b968f67b134184a7853f71c7161f90 2024-11-26T16:42:09,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/96927b4d56e94d529db5a35652f43aad as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/96927b4d56e94d529db5a35652f43aad 2024-11-26T16:42:09,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/96927b4d56e94d529db5a35652f43aad, entries=200, sequenceid=546, filesize=14.4 K 2024-11-26T16:42:09,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/8fcd699ce86941b786b7ff5b5e8c5299 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/8fcd699ce86941b786b7ff5b5e8c5299 2024-11-26T16:42:09,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/8fcd699ce86941b786b7ff5b5e8c5299, entries=150, sequenceid=546, filesize=12.0 K 2024-11-26T16:42:09,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/f1b968f67b134184a7853f71c7161f90 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/f1b968f67b134184a7853f71c7161f90 2024-11-26T16:42:09,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/f1b968f67b134184a7853f71c7161f90, entries=150, sequenceid=546, filesize=12.0 K 2024-11-26T16:42:09,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~435.19 KB/445632, currentSize=26.84 KB/27480 for 8c8b9347f011e2219af1b66f205e204f in 1251ms, sequenceid=546, compaction requested=true 2024-11-26T16:42:09,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:09,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:42:09,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:09,347 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:09,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:42:09,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:09,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c8b9347f011e2219af1b66f205e204f:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:09,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:09,347 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:09,348 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:09,348 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54936 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:09,348 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/A is initiating minor compaction (all files) 2024-11-26T16:42:09,348 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/B is initiating minor compaction (all files) 2024-11-26T16:42:09,348 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/A in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:09,348 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/B in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:09,348 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/a8a500b16d4f43fabb7951af4c814a0b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/874a34dc8ff7480d8f26e6e07491c8e8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/6fa7ad48d77940f59e89463ce6c2c8a0, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/96927b4d56e94d529db5a35652f43aad] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=53.6 K 2024-11-26T16:42:09,348 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a748304b68eb43a3bd76f4b4087fa71b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/0c443cf3b82d42119bce9453a9e1a8e1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/c30a19c0f3844a71923543b79e898674, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/8fcd699ce86941b786b7ff5b5e8c5299] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=48.9 K 2024-11-26T16:42:09,348 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a748304b68eb43a3bd76f4b4087fa71b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732639325308 2024-11-26T16:42:09,348 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8a500b16d4f43fabb7951af4c814a0b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732639325308 2024-11-26T16:42:09,349 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 874a34dc8ff7480d8f26e6e07491c8e8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732639325474 2024-11-26T16:42:09,349 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c443cf3b82d42119bce9453a9e1a8e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732639325493 2024-11-26T16:42:09,349 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting c30a19c0f3844a71923543b79e898674, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1732639326204 2024-11-26T16:42:09,349 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fa7ad48d77940f59e89463ce6c2c8a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1732639326204 2024-11-26T16:42:09,349 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fcd699ce86941b786b7ff5b5e8c5299, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1732639327503 2024-11-26T16:42:09,349 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96927b4d56e94d529db5a35652f43aad, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1732639327503 2024-11-26T16:42:09,358 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#A#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:09,358 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/732dffed5b3d4bacbb6eb66ea0aeed80 is 50, key is test_row_0/A:col10/1732639328094/Put/seqid=0 2024-11-26T16:42:09,359 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#B#compaction#277 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:09,359 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/6eadd87deae846729ae5848fea670bb3 is 50, key is test_row_0/B:col10/1732639328094/Put/seqid=0 2024-11-26T16:42:09,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742144_1320 (size=13289) 2024-11-26T16:42:09,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742145_1321 (size=13289) 2024-11-26T16:42:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-26T16:42:09,671 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:09,672 INFO [Thread-1091 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1091. 2024-11-26T16:42:09,672 DEBUG [Thread-1091 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:42:09,672 DEBUG [Thread-1091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:09,672 DEBUG [Thread-1091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:09,672 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:09,682 INFO [Thread-1089 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1089. 2024-11-26T16:42:09,682 DEBUG [Thread-1089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:42:09,682 DEBUG [Thread-1089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:09,682 DEBUG [Thread-1089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:09,682 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:42:09,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T16:42:09,693 INFO [Thread-1093 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1093. 2024-11-26T16:42:09,693 DEBUG [Thread-1093 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:42:09,693 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:09,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:42:09,693 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:09,693 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:09,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:09,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:42:09,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:09,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:42:09,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:09,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/513af166a1544059a52e26309289ef85 is 50, key is test_row_0/A:col10/1732639329691/Put/seqid=0 2024-11-26T16:42:09,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742146_1322 (size=12301) 2024-11-26T16:42:09,768 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/732dffed5b3d4bacbb6eb66ea0aeed80 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/732dffed5b3d4bacbb6eb66ea0aeed80 2024-11-26T16:42:09,768 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/6eadd87deae846729ae5848fea670bb3 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/6eadd87deae846729ae5848fea670bb3 2024-11-26T16:42:09,772 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/B of 8c8b9347f011e2219af1b66f205e204f into 6eadd87deae846729ae5848fea670bb3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:09,772 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/A of 8c8b9347f011e2219af1b66f205e204f into 732dffed5b3d4bacbb6eb66ea0aeed80(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:09,772 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:09,772 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:09,772 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/B, priority=12, startTime=1732639329347; duration=0sec 2024-11-26T16:42:09,772 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/A, priority=12, startTime=1732639329347; duration=0sec 2024-11-26T16:42:09,772 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:09,772 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:B 2024-11-26T16:42:09,772 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:09,772 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:A 2024-11-26T16:42:09,772 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:09,773 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:09,773 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 8c8b9347f011e2219af1b66f205e204f/C is initiating minor compaction (all files) 2024-11-26T16:42:09,773 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c8b9347f011e2219af1b66f205e204f/C in TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:09,773 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/1e9d65282a484941b4f7c7da09357dd6, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/b957db98d2a94a62afd7fc5709723c78, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/f0d755a8b80245008600434bb69df9d7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/f1b968f67b134184a7853f71c7161f90] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp, totalSize=48.9 K 2024-11-26T16:42:09,774 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e9d65282a484941b4f7c7da09357dd6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732639325308 2024-11-26T16:42:09,774 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting b957db98d2a94a62afd7fc5709723c78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1732639325493 2024-11-26T16:42:09,774 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting f0d755a8b80245008600434bb69df9d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1732639326204 2024-11-26T16:42:09,774 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting f1b968f67b134184a7853f71c7161f90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1732639327503 2024-11-26T16:42:09,781 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c8b9347f011e2219af1b66f205e204f#C#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:09,782 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/5b23580100914136a142f84a47d1e51b is 50, key is test_row_0/C:col10/1732639328094/Put/seqid=0 2024-11-26T16:42:09,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742147_1323 (size=13289) 2024-11-26T16:42:10,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/513af166a1544059a52e26309289ef85 2024-11-26T16:42:10,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/0d2560b226064960a9bd6936dd8c7abc is 50, key is test_row_0/B:col10/1732639329691/Put/seqid=0 2024-11-26T16:42:10,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742148_1324 (size=12301) 2024-11-26T16:42:10,190 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/5b23580100914136a142f84a47d1e51b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/5b23580100914136a142f84a47d1e51b 2024-11-26T16:42:10,194 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8c8b9347f011e2219af1b66f205e204f/C of 8c8b9347f011e2219af1b66f205e204f into 5b23580100914136a142f84a47d1e51b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:10,194 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:10,195 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., storeName=8c8b9347f011e2219af1b66f205e204f/C, priority=12, startTime=1732639329347; duration=0sec 2024-11-26T16:42:10,195 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:10,195 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c8b9347f011e2219af1b66f205e204f:C 2024-11-26T16:42:10,263 INFO [Thread-1087 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1087. 2024-11-26T16:42:10,263 INFO [Thread-1095 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1095. 2024-11-26T16:42:10,263 DEBUG [Thread-1095 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:42:10,263 DEBUG [Thread-1087 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:42:10,263 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:10,263 DEBUG [Thread-1087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:10,263 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:10,263 DEBUG [Thread-1087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:10,263 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:10,263 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:10,263 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T16:42:10,263 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-26T16:42:10,263 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-26T16:42:10,263 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-26T16:42:10,263 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 146 2024-11-26T16:42:10,263 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-26T16:42:10,263 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T16:42:10,263 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3007 2024-11-26T16:42:10,263 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2850 2024-11-26T16:42:10,263 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2786 2024-11-26T16:42:10,263 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3018 2024-11-26T16:42:10,263 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2840 2024-11-26T16:42:10,263 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T16:42:10,264 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T16:42:10,264 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T16:42:10,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:10,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:10,264 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:10,265 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T16:42:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T16:42:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T16:42:10,268 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639330267"}]},"ts":"1732639330267"} 2024-11-26T16:42:10,269 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T16:42:10,269 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T16:42:10,270 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T16:42:10,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8c8b9347f011e2219af1b66f205e204f, UNASSIGN}] 2024-11-26T16:42:10,271 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8c8b9347f011e2219af1b66f205e204f, UNASSIGN 2024-11-26T16:42:10,272 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=8c8b9347f011e2219af1b66f205e204f, regionState=CLOSING, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:10,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8c8b9347f011e2219af1b66f205e204f, UNASSIGN because future has completed 2024-11-26T16:42:10,274 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T16:42:10,274 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134}] 2024-11-26T16:42:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T16:42:10,426 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:42:10,426 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T16:42:10,426 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing 8c8b9347f011e2219af1b66f205e204f, disabling compactions & flushes 2024-11-26T16:42:10,426 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:10,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/0d2560b226064960a9bd6936dd8c7abc 2024-11-26T16:42:10,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/83c4fee784934f2892b4a829d6cd05fd is 50, key is test_row_0/C:col10/1732639329691/Put/seqid=0 2024-11-26T16:42:10,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742149_1325 (size=12301) 2024-11-26T16:42:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T16:42:10,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T16:42:10,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/83c4fee784934f2892b4a829d6cd05fd 2024-11-26T16:42:10,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/513af166a1544059a52e26309289ef85 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/513af166a1544059a52e26309289ef85 2024-11-26T16:42:10,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/513af166a1544059a52e26309289ef85, entries=150, sequenceid=556, filesize=12.0 K 2024-11-26T16:42:10,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/0d2560b226064960a9bd6936dd8c7abc as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/0d2560b226064960a9bd6936dd8c7abc 2024-11-26T16:42:10,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/0d2560b226064960a9bd6936dd8c7abc, entries=150, sequenceid=556, filesize=12.0 K 2024-11-26T16:42:10,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/83c4fee784934f2892b4a829d6cd05fd as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/83c4fee784934f2892b4a829d6cd05fd 2024-11-26T16:42:10,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/83c4fee784934f2892b4a829d6cd05fd, entries=150, sequenceid=556, filesize=12.0 K 2024-11-26T16:42:10,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=13.42 KB/13740 for 8c8b9347f011e2219af1b66f205e204f in 1252ms, sequenceid=556, compaction requested=false 2024-11-26T16:42:10,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c8b9347f011e2219af1b66f205e204f: 2024-11-26T16:42:10,944 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:10,944 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:10,944 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. after waiting 0 ms 2024-11-26T16:42:10,944 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:10,944 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing 8c8b9347f011e2219af1b66f205e204f 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-26T16:42:10,945 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=A 2024-11-26T16:42:10,945 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:10,945 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=B 2024-11-26T16:42:10,945 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:10,945 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8c8b9347f011e2219af1b66f205e204f, store=C 2024-11-26T16:42:10,945 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:10,949 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/2f4daa515016480faacdcb1e387c196d is 50, key is test_row_0/A:col10/1732639330261/Put/seqid=0 2024-11-26T16:42:10,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742150_1326 (size=9857) 2024-11-26T16:42:11,354 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/2f4daa515016480faacdcb1e387c196d 2024-11-26T16:42:11,361 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/2d8e3160bc3143b7b3fe3dbcc050003a is 50, key is test_row_0/B:col10/1732639330261/Put/seqid=0 2024-11-26T16:42:11,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742151_1327 (size=9857) 2024-11-26T16:42:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T16:42:11,765 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/2d8e3160bc3143b7b3fe3dbcc050003a 2024-11-26T16:42:11,773 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/bb3c50270b394288bd3eb437d8a824b5 is 50, key is test_row_0/C:col10/1732639330261/Put/seqid=0 2024-11-26T16:42:11,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742152_1328 (size=9857) 2024-11-26T16:42:11,780 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/bb3c50270b394288bd3eb437d8a824b5 2024-11-26T16:42:11,787 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/A/2f4daa515016480faacdcb1e387c196d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/2f4daa515016480faacdcb1e387c196d 2024-11-26T16:42:11,791 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/2f4daa515016480faacdcb1e387c196d, entries=100, sequenceid=564, filesize=9.6 K 2024-11-26T16:42:11,792 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/B/2d8e3160bc3143b7b3fe3dbcc050003a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/2d8e3160bc3143b7b3fe3dbcc050003a 2024-11-26T16:42:11,796 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/2d8e3160bc3143b7b3fe3dbcc050003a, entries=100, sequenceid=564, filesize=9.6 K 2024-11-26T16:42:11,797 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/.tmp/C/bb3c50270b394288bd3eb437d8a824b5 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/bb3c50270b394288bd3eb437d8a824b5 2024-11-26T16:42:11,801 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/bb3c50270b394288bd3eb437d8a824b5, entries=100, sequenceid=564, filesize=9.6 K 2024-11-26T16:42:11,802 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 8c8b9347f011e2219af1b66f205e204f in 858ms, sequenceid=564, compaction requested=true 2024-11-26T16:42:11,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/0b795fd6e586474a8a955eb519e756b9, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/a9f75cc33df94d0d91e8a0519a0f173b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/06ad7e9e5935428a9052909b9d293538, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/19093a1095a84263abdf35055b5091da, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/6514e557cd4b4b499a6106b0124b8fe1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/cf544b2efa354fe09e0918e3275d94b6, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/514bb8df494845d98f5f37b465bc44fe, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/142ad18243a441b992fdfc9abc0a9b9e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/317218ac3e344aea8a6dc8e676792d50, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/53c1bb8419c6454b8f9ef05dd54f2138, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/20cf12b8fa5d4b18bd67822453d6a534, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/b652e53f9f9041bdae2d65111edfdd25, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/7a568514bb39448ca581f9b9a7c82462, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/7a369f9a0277496fa5d1e1c183b95819, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/5cbd650499a9488ba8a79f763906e620, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/6d1d66bd70b34e139253ae3ae7b8ab8c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/4a229a99a2e940e6aeb5547af437c9cc, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/ae2485b8f23740a3b445fab1dac305a5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/0eb03a2e91904d57a9fb63511af080fc, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/71e7e6ebc8354f0a9e2521f8319a3bd7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/a9c09d412de44839a482f2a7d0c95db7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/f78365b080db44669a251200843e05ba, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/d40986d1d7444617836b29f21524a0ac, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/c9a329e3ff134f98915b65012b715499, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/a8a500b16d4f43fabb7951af4c814a0b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/cc59d30f8d494f7f94fae642c51eecd2, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/874a34dc8ff7480d8f26e6e07491c8e8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/6fa7ad48d77940f59e89463ce6c2c8a0, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/96927b4d56e94d529db5a35652f43aad] to archive 2024-11-26T16:42:11,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T16:42:11,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/0b795fd6e586474a8a955eb519e756b9 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/0b795fd6e586474a8a955eb519e756b9 2024-11-26T16:42:11,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/a9f75cc33df94d0d91e8a0519a0f173b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/a9f75cc33df94d0d91e8a0519a0f173b 2024-11-26T16:42:11,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/06ad7e9e5935428a9052909b9d293538 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/06ad7e9e5935428a9052909b9d293538 2024-11-26T16:42:11,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/19093a1095a84263abdf35055b5091da to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/19093a1095a84263abdf35055b5091da 2024-11-26T16:42:11,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/6514e557cd4b4b499a6106b0124b8fe1 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/6514e557cd4b4b499a6106b0124b8fe1 2024-11-26T16:42:11,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/cf544b2efa354fe09e0918e3275d94b6 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/cf544b2efa354fe09e0918e3275d94b6 2024-11-26T16:42:11,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/514bb8df494845d98f5f37b465bc44fe to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/514bb8df494845d98f5f37b465bc44fe 2024-11-26T16:42:11,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/142ad18243a441b992fdfc9abc0a9b9e to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/142ad18243a441b992fdfc9abc0a9b9e 2024-11-26T16:42:11,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/317218ac3e344aea8a6dc8e676792d50 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/317218ac3e344aea8a6dc8e676792d50 2024-11-26T16:42:11,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/53c1bb8419c6454b8f9ef05dd54f2138 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/53c1bb8419c6454b8f9ef05dd54f2138 2024-11-26T16:42:11,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/20cf12b8fa5d4b18bd67822453d6a534 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/20cf12b8fa5d4b18bd67822453d6a534 2024-11-26T16:42:11,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/b652e53f9f9041bdae2d65111edfdd25 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/b652e53f9f9041bdae2d65111edfdd25 2024-11-26T16:42:11,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/7a568514bb39448ca581f9b9a7c82462 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/7a568514bb39448ca581f9b9a7c82462 2024-11-26T16:42:11,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/7a369f9a0277496fa5d1e1c183b95819 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/7a369f9a0277496fa5d1e1c183b95819 2024-11-26T16:42:11,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/5cbd650499a9488ba8a79f763906e620 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/5cbd650499a9488ba8a79f763906e620 2024-11-26T16:42:11,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/6d1d66bd70b34e139253ae3ae7b8ab8c to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/6d1d66bd70b34e139253ae3ae7b8ab8c 2024-11-26T16:42:11,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/4a229a99a2e940e6aeb5547af437c9cc to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/4a229a99a2e940e6aeb5547af437c9cc 2024-11-26T16:42:11,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/ae2485b8f23740a3b445fab1dac305a5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/ae2485b8f23740a3b445fab1dac305a5 2024-11-26T16:42:11,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/0eb03a2e91904d57a9fb63511af080fc to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/0eb03a2e91904d57a9fb63511af080fc 2024-11-26T16:42:11,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/71e7e6ebc8354f0a9e2521f8319a3bd7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/71e7e6ebc8354f0a9e2521f8319a3bd7 2024-11-26T16:42:11,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/a9c09d412de44839a482f2a7d0c95db7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/a9c09d412de44839a482f2a7d0c95db7 2024-11-26T16:42:11,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/f78365b080db44669a251200843e05ba to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/f78365b080db44669a251200843e05ba 2024-11-26T16:42:11,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/d40986d1d7444617836b29f21524a0ac to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/d40986d1d7444617836b29f21524a0ac 2024-11-26T16:42:11,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/c9a329e3ff134f98915b65012b715499 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/c9a329e3ff134f98915b65012b715499 2024-11-26T16:42:11,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/a8a500b16d4f43fabb7951af4c814a0b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/a8a500b16d4f43fabb7951af4c814a0b 2024-11-26T16:42:11,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/cc59d30f8d494f7f94fae642c51eecd2 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/cc59d30f8d494f7f94fae642c51eecd2 2024-11-26T16:42:11,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/874a34dc8ff7480d8f26e6e07491c8e8 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/874a34dc8ff7480d8f26e6e07491c8e8 2024-11-26T16:42:11,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/6fa7ad48d77940f59e89463ce6c2c8a0 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/6fa7ad48d77940f59e89463ce6c2c8a0 2024-11-26T16:42:11,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/96927b4d56e94d529db5a35652f43aad to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/96927b4d56e94d529db5a35652f43aad 2024-11-26T16:42:11,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a7c3d7e7722f4dbebb4a7198033ad0bc, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/86f6f86ce2d849da9cdc44ecb0094b7d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/45d0bf36ca8247a2bd158fcc8a3c7345, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/702a37b6b55e4b0daac7e389b6b4faa6, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/25faac0e441448a7b0ee48e2c4a6667d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/46f161bfe85d461e9ae896e6cdaad0d1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/19b0d0e8c7f847d1b90aa46101d0996b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/f8cf776b7e4d4675afe60c0703c2963a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/59249f6d849a4e148940a508628b632a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a9f33e211fe2440ebdd79c68bb6893ab, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/4945799070244047b7d7a4cb248c2d71, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/5c2a48bdfc6047fd8e2431818dad44fa, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/4379fb32d4e04eb6800dbf04bd8e118b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/fc2947dadc924ca1955b744eb47ccc21, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/0aeee3db914744939f129e98fe85803a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/dd86959c984649a5b65178ab20bb6c72, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/3881af98b30a4e23b001f8ff97884df4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/b6da699de4f545f6a8dd1ff8ac41022a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/49fcfdf11b7c42ad973e15475466c5ad, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/5d98f077f0b145dea4630763287b7141, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a66b0623db3343c6a12c53cea71bf490, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a52bec8434394c089bcb9e00a5c631b7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/41cd4541d91f4601ac37e858f4651a8a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/d3a499e4d1e54b8f9dbf6b49f6f7592e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a748304b68eb43a3bd76f4b4087fa71b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/3c4b6b774773474881335cd6ad1de12a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/0c443cf3b82d42119bce9453a9e1a8e1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/c30a19c0f3844a71923543b79e898674, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/8fcd699ce86941b786b7ff5b5e8c5299] to archive 2024-11-26T16:42:11,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T16:42:11,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a7c3d7e7722f4dbebb4a7198033ad0bc to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a7c3d7e7722f4dbebb4a7198033ad0bc 2024-11-26T16:42:11,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/86f6f86ce2d849da9cdc44ecb0094b7d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/86f6f86ce2d849da9cdc44ecb0094b7d 2024-11-26T16:42:11,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/45d0bf36ca8247a2bd158fcc8a3c7345 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/45d0bf36ca8247a2bd158fcc8a3c7345 2024-11-26T16:42:11,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/702a37b6b55e4b0daac7e389b6b4faa6 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/702a37b6b55e4b0daac7e389b6b4faa6 2024-11-26T16:42:11,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/25faac0e441448a7b0ee48e2c4a6667d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/25faac0e441448a7b0ee48e2c4a6667d 2024-11-26T16:42:11,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/46f161bfe85d461e9ae896e6cdaad0d1 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/46f161bfe85d461e9ae896e6cdaad0d1 2024-11-26T16:42:11,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/19b0d0e8c7f847d1b90aa46101d0996b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/19b0d0e8c7f847d1b90aa46101d0996b 2024-11-26T16:42:11,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/f8cf776b7e4d4675afe60c0703c2963a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/f8cf776b7e4d4675afe60c0703c2963a 2024-11-26T16:42:11,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/59249f6d849a4e148940a508628b632a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/59249f6d849a4e148940a508628b632a 2024-11-26T16:42:11,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a9f33e211fe2440ebdd79c68bb6893ab to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a9f33e211fe2440ebdd79c68bb6893ab 2024-11-26T16:42:11,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/4945799070244047b7d7a4cb248c2d71 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/4945799070244047b7d7a4cb248c2d71 2024-11-26T16:42:11,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/5c2a48bdfc6047fd8e2431818dad44fa to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/5c2a48bdfc6047fd8e2431818dad44fa 2024-11-26T16:42:11,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/4379fb32d4e04eb6800dbf04bd8e118b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/4379fb32d4e04eb6800dbf04bd8e118b 2024-11-26T16:42:11,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/fc2947dadc924ca1955b744eb47ccc21 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/fc2947dadc924ca1955b744eb47ccc21 2024-11-26T16:42:11,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/0aeee3db914744939f129e98fe85803a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/0aeee3db914744939f129e98fe85803a 2024-11-26T16:42:11,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/dd86959c984649a5b65178ab20bb6c72 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/dd86959c984649a5b65178ab20bb6c72 2024-11-26T16:42:11,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/3881af98b30a4e23b001f8ff97884df4 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/3881af98b30a4e23b001f8ff97884df4 2024-11-26T16:42:11,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/b6da699de4f545f6a8dd1ff8ac41022a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/b6da699de4f545f6a8dd1ff8ac41022a 2024-11-26T16:42:11,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/49fcfdf11b7c42ad973e15475466c5ad to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/49fcfdf11b7c42ad973e15475466c5ad 2024-11-26T16:42:11,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/5d98f077f0b145dea4630763287b7141 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/5d98f077f0b145dea4630763287b7141 2024-11-26T16:42:11,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a66b0623db3343c6a12c53cea71bf490 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a66b0623db3343c6a12c53cea71bf490 2024-11-26T16:42:11,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a52bec8434394c089bcb9e00a5c631b7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a52bec8434394c089bcb9e00a5c631b7 2024-11-26T16:42:11,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/41cd4541d91f4601ac37e858f4651a8a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/41cd4541d91f4601ac37e858f4651a8a 2024-11-26T16:42:11,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/d3a499e4d1e54b8f9dbf6b49f6f7592e to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/d3a499e4d1e54b8f9dbf6b49f6f7592e 2024-11-26T16:42:11,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a748304b68eb43a3bd76f4b4087fa71b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/a748304b68eb43a3bd76f4b4087fa71b 2024-11-26T16:42:11,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/3c4b6b774773474881335cd6ad1de12a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/3c4b6b774773474881335cd6ad1de12a 2024-11-26T16:42:11,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/0c443cf3b82d42119bce9453a9e1a8e1 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/0c443cf3b82d42119bce9453a9e1a8e1 2024-11-26T16:42:11,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/c30a19c0f3844a71923543b79e898674 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/c30a19c0f3844a71923543b79e898674 2024-11-26T16:42:11,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/8fcd699ce86941b786b7ff5b5e8c5299 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/8fcd699ce86941b786b7ff5b5e8c5299 2024-11-26T16:42:11,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/a22304964d794101b872cbc50a16d386, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/3cf658b406fd43fbbed930d90560ca82, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/0f055f581bc840768a18e5d6c927f9d5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/947086e2e48c45be87d4903fff0e6afe, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/60bca55dbe7140019d49ec776e2a3538, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/de09fc821e1c46c4846e63a6d89d81c8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/df1c0e5982de49be9bd64926a417192e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/89c206f5dc594ac092a0294daeb21c7a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/8ad0c6dfd06247c580f711c43ef7aac5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/49766e5ff9004220be419f0c14c5c9af, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/903ac39919c94c66b72cce91d2743928, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/5eabfe38d73441a3a29762de22b5c9a7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/5fed311d528842e28381628e8d4620b5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/aad9b71bb1654bf591b993960311dc2b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/00815924c3e44050a60468afa49919f6, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/3498596f87d14418a29e2fe21d9787e9, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/f95653e7c01d4ac19b3a83258e582ecd, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/058baad17f0e4d329a774c51e34f6348, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/ba92045f39294d89b84375cb6bcec489, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/2f005e49ecdc4421960f71561fdd3dcf, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/e4692c19ad51401488acd8a5b39e9ea7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/71a1ef0d2f9143b483d0ca4a2ea90dc9, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/6803605bd57e454a8a3b4e1baac17439, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/9cad54294c0b4e53b2706c6cd142d343, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/1e9d65282a484941b4f7c7da09357dd6, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/784abc55f08a464381b589c044b3cdd0, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/b957db98d2a94a62afd7fc5709723c78, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/f0d755a8b80245008600434bb69df9d7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/f1b968f67b134184a7853f71c7161f90] to archive 2024-11-26T16:42:11,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T16:42:11,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/a22304964d794101b872cbc50a16d386 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/a22304964d794101b872cbc50a16d386 2024-11-26T16:42:11,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/3cf658b406fd43fbbed930d90560ca82 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/3cf658b406fd43fbbed930d90560ca82 2024-11-26T16:42:11,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/0f055f581bc840768a18e5d6c927f9d5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/0f055f581bc840768a18e5d6c927f9d5 2024-11-26T16:42:11,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/947086e2e48c45be87d4903fff0e6afe to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/947086e2e48c45be87d4903fff0e6afe 2024-11-26T16:42:11,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/60bca55dbe7140019d49ec776e2a3538 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/60bca55dbe7140019d49ec776e2a3538 2024-11-26T16:42:11,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/de09fc821e1c46c4846e63a6d89d81c8 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/de09fc821e1c46c4846e63a6d89d81c8 2024-11-26T16:42:11,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/df1c0e5982de49be9bd64926a417192e to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/df1c0e5982de49be9bd64926a417192e 2024-11-26T16:42:11,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/89c206f5dc594ac092a0294daeb21c7a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/89c206f5dc594ac092a0294daeb21c7a 2024-11-26T16:42:11,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/8ad0c6dfd06247c580f711c43ef7aac5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/8ad0c6dfd06247c580f711c43ef7aac5 2024-11-26T16:42:11,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/49766e5ff9004220be419f0c14c5c9af to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/49766e5ff9004220be419f0c14c5c9af 2024-11-26T16:42:11,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/903ac39919c94c66b72cce91d2743928 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/903ac39919c94c66b72cce91d2743928 2024-11-26T16:42:11,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/5eabfe38d73441a3a29762de22b5c9a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/5eabfe38d73441a3a29762de22b5c9a7 2024-11-26T16:42:11,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/5fed311d528842e28381628e8d4620b5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/5fed311d528842e28381628e8d4620b5 2024-11-26T16:42:11,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/aad9b71bb1654bf591b993960311dc2b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/aad9b71bb1654bf591b993960311dc2b 2024-11-26T16:42:11,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/00815924c3e44050a60468afa49919f6 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/00815924c3e44050a60468afa49919f6 2024-11-26T16:42:11,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/3498596f87d14418a29e2fe21d9787e9 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/3498596f87d14418a29e2fe21d9787e9 2024-11-26T16:42:11,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/f95653e7c01d4ac19b3a83258e582ecd to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/f95653e7c01d4ac19b3a83258e582ecd 2024-11-26T16:42:11,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/058baad17f0e4d329a774c51e34f6348 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/058baad17f0e4d329a774c51e34f6348 2024-11-26T16:42:11,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/ba92045f39294d89b84375cb6bcec489 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/ba92045f39294d89b84375cb6bcec489 2024-11-26T16:42:11,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/2f005e49ecdc4421960f71561fdd3dcf to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/2f005e49ecdc4421960f71561fdd3dcf 2024-11-26T16:42:11,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/e4692c19ad51401488acd8a5b39e9ea7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/e4692c19ad51401488acd8a5b39e9ea7 2024-11-26T16:42:11,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/71a1ef0d2f9143b483d0ca4a2ea90dc9 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/71a1ef0d2f9143b483d0ca4a2ea90dc9 2024-11-26T16:42:11,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/6803605bd57e454a8a3b4e1baac17439 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/6803605bd57e454a8a3b4e1baac17439 2024-11-26T16:42:11,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/9cad54294c0b4e53b2706c6cd142d343 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/9cad54294c0b4e53b2706c6cd142d343 2024-11-26T16:42:11,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/1e9d65282a484941b4f7c7da09357dd6 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/1e9d65282a484941b4f7c7da09357dd6 2024-11-26T16:42:11,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/784abc55f08a464381b589c044b3cdd0 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/784abc55f08a464381b589c044b3cdd0 2024-11-26T16:42:11,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/b957db98d2a94a62afd7fc5709723c78 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/b957db98d2a94a62afd7fc5709723c78 2024-11-26T16:42:11,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/f0d755a8b80245008600434bb69df9d7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/f0d755a8b80245008600434bb69df9d7 2024-11-26T16:42:11,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/f1b968f67b134184a7853f71c7161f90 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/f1b968f67b134184a7853f71c7161f90 2024-11-26T16:42:12,004 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/recovered.edits/567.seqid, newMaxSeqId=567, maxSeqId=1 2024-11-26T16:42:12,005 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. 2024-11-26T16:42:12,005 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for 8c8b9347f011e2219af1b66f205e204f: Waiting for close lock at 1732639330426Running coprocessor pre-close hooks at 1732639330426Disabling compacts and flushes for region at 1732639330426Disabling writes for close at 1732639330944 (+518 ms)Obtaining lock to block concurrent updates at 1732639330944Preparing flush snapshotting stores in 8c8b9347f011e2219af1b66f205e204f at 1732639330944Finished memstore snapshotting TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732639330945 (+1 ms)Flushing stores of TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f. at 1732639330945Flushing 8c8b9347f011e2219af1b66f205e204f/A: creating writer at 1732639330945Flushing 8c8b9347f011e2219af1b66f205e204f/A: appending metadata at 1732639330948 (+3 ms)Flushing 8c8b9347f011e2219af1b66f205e204f/A: closing flushed file at 1732639330948Flushing 8c8b9347f011e2219af1b66f205e204f/B: creating writer at 1732639331358 (+410 ms)Flushing 8c8b9347f011e2219af1b66f205e204f/B: appending metadata at 1732639331360 (+2 ms)Flushing 8c8b9347f011e2219af1b66f205e204f/B: closing flushed file at 1732639331361 (+1 ms)Flushing 8c8b9347f011e2219af1b66f205e204f/C: creating writer at 1732639331770 (+409 ms)Flushing 8c8b9347f011e2219af1b66f205e204f/C: appending metadata at 1732639331772 (+2 ms)Flushing 8c8b9347f011e2219af1b66f205e204f/C: closing flushed file at 1732639331772Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63eb8dd1: reopening flushed file at 1732639331786 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c4b5362: reopening flushed file at 1732639331792 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40728c22: reopening flushed file at 1732639331796 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 8c8b9347f011e2219af1b66f205e204f in 858ms, sequenceid=564, compaction requested=true at 1732639331802 (+6 ms)Writing region close event to WAL at 1732639331980 (+178 ms)Running coprocessor post-close hooks at 1732639332005 (+25 ms)Closed at 1732639332005 2024-11-26T16:42:12,008 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed 8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:42:12,009 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=8c8b9347f011e2219af1b66f205e204f, regionState=CLOSED 2024-11-26T16:42:12,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 because future has completed 2024-11-26T16:42:12,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-26T16:42:12,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure 8c8b9347f011e2219af1b66f205e204f, server=2dedfe7466e8,38109,1732639238134 in 1.7380 sec 2024-11-26T16:42:12,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-26T16:42:12,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8c8b9347f011e2219af1b66f205e204f, UNASSIGN in 1.7430 sec 2024-11-26T16:42:12,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-26T16:42:12,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7480 sec 2024-11-26T16:42:12,021 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639332020"}]},"ts":"1732639332020"} 2024-11-26T16:42:12,022 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T16:42:12,022 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T16:42:12,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7580 sec 2024-11-26T16:42:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-26T16:42:12,412 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:12,413 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T16:42:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:42:12,415 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:42:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-26T16:42:12,416 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:42:12,418 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:42:12,421 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A, FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B, FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C, FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/recovered.edits] 2024-11-26T16:42:12,427 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/2f4daa515016480faacdcb1e387c196d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/2f4daa515016480faacdcb1e387c196d 2024-11-26T16:42:12,428 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/513af166a1544059a52e26309289ef85 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/513af166a1544059a52e26309289ef85 2024-11-26T16:42:12,429 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/732dffed5b3d4bacbb6eb66ea0aeed80 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/A/732dffed5b3d4bacbb6eb66ea0aeed80 2024-11-26T16:42:12,432 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/0d2560b226064960a9bd6936dd8c7abc to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/0d2560b226064960a9bd6936dd8c7abc 2024-11-26T16:42:12,434 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/2d8e3160bc3143b7b3fe3dbcc050003a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/2d8e3160bc3143b7b3fe3dbcc050003a 2024-11-26T16:42:12,435 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/6eadd87deae846729ae5848fea670bb3 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/B/6eadd87deae846729ae5848fea670bb3 2024-11-26T16:42:12,438 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/5b23580100914136a142f84a47d1e51b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/5b23580100914136a142f84a47d1e51b 2024-11-26T16:42:12,439 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/83c4fee784934f2892b4a829d6cd05fd to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/83c4fee784934f2892b4a829d6cd05fd 2024-11-26T16:42:12,441 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/bb3c50270b394288bd3eb437d8a824b5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/C/bb3c50270b394288bd3eb437d8a824b5 2024-11-26T16:42:12,445 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/recovered.edits/567.seqid to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f/recovered.edits/567.seqid 2024-11-26T16:42:12,445 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/8c8b9347f011e2219af1b66f205e204f 2024-11-26T16:42:12,445 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T16:42:12,448 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:42:12,450 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T16:42:12,453 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T16:42:12,454 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:42:12,454 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T16:42:12,454 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732639332454"}]},"ts":"9223372036854775807"} 2024-11-26T16:42:12,457 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T16:42:12,457 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8c8b9347f011e2219af1b66f205e204f, NAME => 'TestAcidGuarantees,,1732639306231.8c8b9347f011e2219af1b66f205e204f.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T16:42:12,457 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T16:42:12,458 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732639332457"}]},"ts":"9223372036854775807"} 2024-11-26T16:42:12,461 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T16:42:12,461 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:42:12,463 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-11-26T16:42:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-26T16:42:12,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T16:42:12,524 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:12,532 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2059946220_22 at /127.0.0.1:55688 [Waiting for operation #359] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1947582375_22 at /127.0.0.1:54996 [Waiting for operation #228] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=403 (was 447), ProcessCount=11 (was 11), AvailableMemoryMB=5312 (was 5949) 2024-11-26T16:42:12,544 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=403, ProcessCount=11, AvailableMemoryMB=5311 2024-11-26T16:42:12,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T16:42:12,546 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T16:42:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T16:42:12,548 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T16:42:12,549 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:12,549 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-26T16:42:12,550 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T16:42:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T16:42:12,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742153_1329 (size=1061) 2024-11-26T16:42:12,561 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711 2024-11-26T16:42:12,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742154_1330 (size=53) 2024-11-26T16:42:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T16:42:12,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T16:42:12,970 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T16:42:12,970 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 28a7a40a9a40603189977c8da6b1d6a7, disabling compactions & flushes 2024-11-26T16:42:12,970 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:12,970 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:12,970 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. after waiting 0 ms 2024-11-26T16:42:12,970 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:12,970 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:12,970 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 28a7a40a9a40603189977c8da6b1d6a7: Waiting for close lock at 1732639332970Disabling compacts and flushes for region at 1732639332970Disabling writes for close at 1732639332970Writing region close event to WAL at 1732639332970Closed at 1732639332970 2024-11-26T16:42:12,971 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T16:42:12,972 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732639332971"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732639332971"}]},"ts":"1732639332971"} 2024-11-26T16:42:12,973 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T16:42:12,974 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T16:42:12,974 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639332974"}]},"ts":"1732639332974"} 2024-11-26T16:42:12,975 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T16:42:12,975 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=28a7a40a9a40603189977c8da6b1d6a7, ASSIGN}] 2024-11-26T16:42:12,976 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=28a7a40a9a40603189977c8da6b1d6a7, ASSIGN 2024-11-26T16:42:12,977 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=28a7a40a9a40603189977c8da6b1d6a7, ASSIGN; state=OFFLINE, location=2dedfe7466e8,38109,1732639238134; forceNewPlan=false, retain=false 2024-11-26T16:42:13,128 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=28a7a40a9a40603189977c8da6b1d6a7, regionState=OPENING, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:13,129 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=28a7a40a9a40603189977c8da6b1d6a7, ASSIGN because future has completed 2024-11-26T16:42:13,130 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134}] 2024-11-26T16:42:13,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T16:42:13,284 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:13,284 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} 2024-11-26T16:42:13,285 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:13,285 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T16:42:13,285 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:13,285 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:13,286 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:13,287 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:42:13,287 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28a7a40a9a40603189977c8da6b1d6a7 columnFamilyName A 2024-11-26T16:42:13,287 DEBUG [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:13,288 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] regionserver.HStore(327): Store=28a7a40a9a40603189977c8da6b1d6a7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:42:13,288 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:13,289 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:42:13,289 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28a7a40a9a40603189977c8da6b1d6a7 columnFamilyName B 2024-11-26T16:42:13,289 DEBUG [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:13,289 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] regionserver.HStore(327): Store=28a7a40a9a40603189977c8da6b1d6a7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:42:13,289 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:13,290 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:42:13,290 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28a7a40a9a40603189977c8da6b1d6a7 columnFamilyName C 2024-11-26T16:42:13,290 DEBUG [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:13,291 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] regionserver.HStore(327): Store=28a7a40a9a40603189977c8da6b1d6a7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:42:13,291 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:13,291 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:13,291 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:13,291 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:13,292 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:13,292 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:13,293 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T16:42:13,293 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:13,295 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T16:42:13,295 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 28a7a40a9a40603189977c8da6b1d6a7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70186114, jitterRate=0.0458545982837677}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T16:42:13,295 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:13,296 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 28a7a40a9a40603189977c8da6b1d6a7: Running coprocessor pre-open hook at 1732639333285Writing region info on filesystem at 1732639333285Initializing all the Stores at 1732639333286 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639333286Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639333286Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639333286Cleaning up temporary data from old regions at 1732639333292 (+6 ms)Running coprocessor post-open hooks at 1732639333295 (+3 ms)Region opened successfully at 1732639333296 (+1 ms) 2024-11-26T16:42:13,297 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., pid=87, masterSystemTime=1732639333281 2024-11-26T16:42:13,299 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:13,299 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:13,299 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=28a7a40a9a40603189977c8da6b1d6a7, regionState=OPEN, openSeqNum=2, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:13,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 because future has completed 2024-11-26T16:42:13,302 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-26T16:42:13,302 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 in 172 msec 2024-11-26T16:42:13,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-26T16:42:13,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=28a7a40a9a40603189977c8da6b1d6a7, ASSIGN in 328 msec 2024-11-26T16:42:13,305 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T16:42:13,305 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639333305"}]},"ts":"1732639333305"} 2024-11-26T16:42:13,306 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T16:42:13,307 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T16:42:13,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 761 msec 2024-11-26T16:42:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-26T16:42:13,691 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:13,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24132877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:13,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:13,693 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:13,694 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23b5a2cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:13,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:13,695 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:13,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a40d8ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:13,696 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:13,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:13,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:13,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:13,698 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T16:42:13,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 2dedfe7466e8,35233,1732639237322 2024-11-26T16:42:13,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1d8a9e2b 2024-11-26T16:42:13,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T16:42:13,700 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T16:42:13,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T16:42:13,701 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T16:42:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T16:42:13,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742155_1331 (size=1097) 2024-11-26T16:42:14,113 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-26T16:42:14,113 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-26T16:42:14,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T16:42:14,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=28a7a40a9a40603189977c8da6b1d6a7, REOPEN/MOVE}] 2024-11-26T16:42:14,118 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=28a7a40a9a40603189977c8da6b1d6a7, REOPEN/MOVE 2024-11-26T16:42:14,118 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=28a7a40a9a40603189977c8da6b1d6a7, regionState=CLOSING, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:14,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=28a7a40a9a40603189977c8da6b1d6a7, REOPEN/MOVE because future has completed 2024-11-26T16:42:14,120 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T16:42:14,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134}] 2024-11-26T16:42:14,272 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:14,272 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T16:42:14,272 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 28a7a40a9a40603189977c8da6b1d6a7, disabling compactions & flushes 2024-11-26T16:42:14,272 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:14,272 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:14,272 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. after waiting 0 ms 2024-11-26T16:42:14,272 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:14,276 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T16:42:14,277 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:14,277 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 28a7a40a9a40603189977c8da6b1d6a7: Waiting for close lock at 1732639334272Running coprocessor pre-close hooks at 1732639334272Disabling compacts and flushes for region at 1732639334272Disabling writes for close at 1732639334272Writing region close event to WAL at 1732639334273 (+1 ms)Running coprocessor post-close hooks at 1732639334276 (+3 ms)Closed at 1732639334276 2024-11-26T16:42:14,277 WARN [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: 28a7a40a9a40603189977c8da6b1d6a7 to self. 2024-11-26T16:42:14,279 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:14,279 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=28a7a40a9a40603189977c8da6b1d6a7, regionState=CLOSED 2024-11-26T16:42:14,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 because future has completed 2024-11-26T16:42:14,282 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-26T16:42:14,282 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 in 161 msec 2024-11-26T16:42:14,283 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=28a7a40a9a40603189977c8da6b1d6a7, REOPEN/MOVE; state=CLOSED, location=2dedfe7466e8,38109,1732639238134; forceNewPlan=false, retain=true 2024-11-26T16:42:14,434 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=28a7a40a9a40603189977c8da6b1d6a7, regionState=OPENING, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:14,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=28a7a40a9a40603189977c8da6b1d6a7, REOPEN/MOVE because future has completed 2024-11-26T16:42:14,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134}] 2024-11-26T16:42:14,590 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:14,590 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} 2024-11-26T16:42:14,590 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:14,591 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T16:42:14,591 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:14,591 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:14,592 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:14,593 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:42:14,593 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28a7a40a9a40603189977c8da6b1d6a7 columnFamilyName A 2024-11-26T16:42:14,594 DEBUG [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:14,594 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] regionserver.HStore(327): Store=28a7a40a9a40603189977c8da6b1d6a7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:42:14,595 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:14,595 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:42:14,595 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28a7a40a9a40603189977c8da6b1d6a7 columnFamilyName B 2024-11-26T16:42:14,595 DEBUG [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:14,596 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] regionserver.HStore(327): Store=28a7a40a9a40603189977c8da6b1d6a7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:42:14,596 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:14,596 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:42:14,596 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28a7a40a9a40603189977c8da6b1d6a7 columnFamilyName C 2024-11-26T16:42:14,596 DEBUG [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:14,597 INFO [StoreOpener-28a7a40a9a40603189977c8da6b1d6a7-1 {}] regionserver.HStore(327): Store=28a7a40a9a40603189977c8da6b1d6a7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:42:14,597 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:14,597 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:14,597 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:14,598 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:14,599 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:14,599 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:14,600 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T16:42:14,601 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:14,601 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened 28a7a40a9a40603189977c8da6b1d6a7; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67281420, jitterRate=0.002571284770965576}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T16:42:14,601 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:14,602 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for 28a7a40a9a40603189977c8da6b1d6a7: Running coprocessor pre-open hook at 1732639334591Writing region info on filesystem at 1732639334591Initializing all the Stores at 1732639334591Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639334591Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639334592 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639334592Cleaning up temporary data from old regions at 1732639334599 (+7 ms)Running coprocessor post-open hooks at 1732639334601 (+2 ms)Region opened successfully at 1732639334602 (+1 ms) 2024-11-26T16:42:14,603 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., pid=92, masterSystemTime=1732639334587 2024-11-26T16:42:14,604 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:14,605 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:14,605 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=28a7a40a9a40603189977c8da6b1d6a7, regionState=OPEN, openSeqNum=5, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:14,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 because future has completed 2024-11-26T16:42:14,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-26T16:42:14,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 in 171 msec 2024-11-26T16:42:14,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-26T16:42:14,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=28a7a40a9a40603189977c8da6b1d6a7, REOPEN/MOVE in 492 msec 2024-11-26T16:42:14,612 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-26T16:42:14,612 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 496 msec 2024-11-26T16:42:14,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 911 msec 2024-11-26T16:42:14,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-26T16:42:14,615 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:14,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14572f5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,616 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:14,616 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:14,617 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:14,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec0772b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:14,619 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15ff66fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38802cb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,620 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:14,620 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:14,621 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f4f5df0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:14,622 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a97cc02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@209ba84b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,623 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:14,624 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:14,624 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:14,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@666f3e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:14,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:14,625 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35195fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7495e76b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,626 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:14,626 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:14,627 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@695e212f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:14,628 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41ba2293, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7288b4d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,629 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:14,629 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:14,630 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@527d4f5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:14,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:14,631 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:14,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@127f5929, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50946179, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,632 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:14,632 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:14,633 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@741c2280, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:14,634 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:14,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@361be5e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62931224, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,635 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:14,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:14,636 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@482db613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:14,637 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:14,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@723ea378, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@400fc310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,638 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:14,638 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:14,638 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:14,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:14,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:14,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1303cfad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:14,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:14,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:14,640 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:14,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e54d8c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@227d6625, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,641 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:14,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:14,641 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57d33abf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:14,643 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:14,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@363b9f2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15113514, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,644 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:14,644 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:14,644 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cea5205, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:14,646 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21837aa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:14,647 DEBUG [Thread-1522 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:14,647 DEBUG [Thread-1530 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:14,647 DEBUG [Thread-1513 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:14,647 DEBUG [Thread-1511 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:14,647 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:42:14,648 DEBUG [Thread-1519 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:14,648 DEBUG [Thread-1526 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:14,648 DEBUG [Thread-1515 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-26T16:42:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:14,649 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:14,649 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:14,649 DEBUG [Thread-1524 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:14,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:14,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:14,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:14,650 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:42:14,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:14,650 DEBUG [Thread-1524 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:14,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:14,651 DEBUG [Thread-1524 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:14,651 DEBUG [Thread-1515 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:14,651 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T16:42:14,651 DEBUG [Thread-1515 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:14,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:14,652 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:42:14,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:14,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:42:14,652 DEBUG [Thread-1528 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:42:14,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:42:14,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:14,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:14,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:42:14,655 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:42:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:42:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:42:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:42:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:42:14,658 DEBUG [Thread-1517 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:14,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:42:14,659 DEBUG [Thread-1517 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:14,660 DEBUG [Thread-1517 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:14,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:42:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:14,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28a7a40a9a40603189977c8da6b1d6a7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T16:42:14,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=A 2024-11-26T16:42:14,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:14,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=B 2024-11-26T16:42:14,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:14,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=C 2024-11-26T16:42:14,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:14,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112647e471d30b884e87a2102b10681cd1e1_28a7a40a9a40603189977c8da6b1d6a7 is 50, key is test_row_0/A:col10/1732639334662/Put/seqid=0 2024-11-26T16:42:14,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742156_1332 (size=14594) 2024-11-26T16:42:14,706 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:14,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732639334673, max timestamp=1732639334707Num uniques 859; 2024-11-26T16:42:14,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-26T16:42:14,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732639334673, max timestamp=1732639334707Num uniques 859; 2024-11-26T16:42:14,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-26T16:42:14,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732639334673, max timestamp=1732639334707Num uniques 859; 2024-11-26T16:42:14,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-26T16:42:14,714 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112647e471d30b884e87a2102b10681cd1e1_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112647e471d30b884e87a2102b10681cd1e1_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:14,715 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/5be0f4eb416547f2a437b8339cd0d534, store: [table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:14,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/5be0f4eb416547f2a437b8339cd0d534 is 175, key is test_row_0/A:col10/1732639334662/Put/seqid=0 2024-11-26T16:42:14,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742157_1333 (size=39549) 2024-11-26T16:42:14,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639394739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:14,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639394742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:14,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:14,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639394743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:14,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639394751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T16:42:14,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:14,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639394752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:14,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-26T16:42:14,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:14,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:14,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:14,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:14,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:14,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:14,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639394861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:14,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:14,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:14,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:14,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639394862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:14,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:14,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:14,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:14,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639394862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:14,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:14,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:14,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:14,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639394862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:14,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:14,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:14,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:14,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:14,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:14,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639394871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:14,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:14,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:14,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:14,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-26T16:42:14,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:14,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:14,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:14,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:14,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T16:42:15,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639395071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:15,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:15,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639395071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:15,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:15,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639395071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639395071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:15,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639395090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:15,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-26T16:42:15,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:15,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:15,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:15,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:15,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:15,127 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/5be0f4eb416547f2a437b8339cd0d534 2024-11-26T16:42:15,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/560ccab703b14672ac4ffccc7cea038b is 50, key is test_row_0/B:col10/1732639334662/Put/seqid=0 2024-11-26T16:42:15,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742158_1334 (size=12001) 2024-11-26T16:42:15,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/560ccab703b14672ac4ffccc7cea038b 2024-11-26T16:42:15,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/9b818ef5b36e468d8c96e028602f2ed4 is 50, key is test_row_0/C:col10/1732639334662/Put/seqid=0 2024-11-26T16:42:15,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742159_1335 (size=12001) 2024-11-26T16:42:15,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/9b818ef5b36e468d8c96e028602f2ed4 2024-11-26T16:42:15,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/5be0f4eb416547f2a437b8339cd0d534 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/5be0f4eb416547f2a437b8339cd0d534 2024-11-26T16:42:15,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/5be0f4eb416547f2a437b8339cd0d534, entries=200, sequenceid=14, filesize=38.6 K 2024-11-26T16:42:15,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/560ccab703b14672ac4ffccc7cea038b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/560ccab703b14672ac4ffccc7cea038b 2024-11-26T16:42:15,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/560ccab703b14672ac4ffccc7cea038b, entries=150, sequenceid=14, filesize=11.7 K 2024-11-26T16:42:15,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/9b818ef5b36e468d8c96e028602f2ed4 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/9b818ef5b36e468d8c96e028602f2ed4 2024-11-26T16:42:15,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/9b818ef5b36e468d8c96e028602f2ed4, entries=150, sequenceid=14, filesize=11.7 K 2024-11-26T16:42:15,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 28a7a40a9a40603189977c8da6b1d6a7 in 540ms, sequenceid=14, compaction requested=false 2024-11-26T16:42:15,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:15,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-26T16:42:15,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:15,266 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 28a7a40a9a40603189977c8da6b1d6a7 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-26T16:42:15,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=A 2024-11-26T16:42:15,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:15,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=B 2024-11-26T16:42:15,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:15,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=C 2024-11-26T16:42:15,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:15,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112694fc7b7e1cda485ba1aff797cf470b2d_28a7a40a9a40603189977c8da6b1d6a7 is 50, key is test_row_0/A:col10/1732639334742/Put/seqid=0 2024-11-26T16:42:15,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742160_1336 (size=12154) 2024-11-26T16:42:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T16:42:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:15,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:15,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639395402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:15,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:15,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639395403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:15,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:15,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639395411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:15,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639395413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:15,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:15,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639395414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:15,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:15,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:15,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639395521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:15,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639395521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:15,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639395531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:15,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639395531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:15,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:15,686 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112694fc7b7e1cda485ba1aff797cf470b2d_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112694fc7b7e1cda485ba1aff797cf470b2d_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:15,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/4a0d00113f99499ca67027eb1f013f11, store: [table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:15,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/4a0d00113f99499ca67027eb1f013f11 is 175, key is test_row_0/A:col10/1732639334742/Put/seqid=0 2024-11-26T16:42:15,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742161_1337 (size=30955) 2024-11-26T16:42:15,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:15,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639395741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:15,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:15,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:15,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639395741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:15,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639395751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:15,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639395751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T16:42:15,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639395931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:16,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:16,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639396061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:16,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:16,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639396061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:16,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:16,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:16,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639396071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:16,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639396071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:16,092 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/4a0d00113f99499ca67027eb1f013f11 2024-11-26T16:42:16,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/41f26ffe39374165a7dc76b4aa3af301 is 50, key is test_row_0/B:col10/1732639334742/Put/seqid=0 2024-11-26T16:42:16,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742162_1338 (size=12001) 2024-11-26T16:42:16,106 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/41f26ffe39374165a7dc76b4aa3af301 2024-11-26T16:42:16,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/a48c36efa2454fe295386b2edfac4895 is 50, key is test_row_0/C:col10/1732639334742/Put/seqid=0 2024-11-26T16:42:16,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742163_1339 (size=12001) 2024-11-26T16:42:16,519 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/a48c36efa2454fe295386b2edfac4895 2024-11-26T16:42:16,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/4a0d00113f99499ca67027eb1f013f11 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4a0d00113f99499ca67027eb1f013f11 2024-11-26T16:42:16,527 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4a0d00113f99499ca67027eb1f013f11, entries=150, sequenceid=48, filesize=30.2 K 2024-11-26T16:42:16,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/41f26ffe39374165a7dc76b4aa3af301 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/41f26ffe39374165a7dc76b4aa3af301 2024-11-26T16:42:16,532 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/41f26ffe39374165a7dc76b4aa3af301, entries=150, sequenceid=48, filesize=11.7 K 2024-11-26T16:42:16,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/a48c36efa2454fe295386b2edfac4895 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/a48c36efa2454fe295386b2edfac4895 2024-11-26T16:42:16,536 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/a48c36efa2454fe295386b2edfac4895, entries=150, sequenceid=48, filesize=11.7 K 2024-11-26T16:42:16,537 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for 28a7a40a9a40603189977c8da6b1d6a7 in 1270ms, sequenceid=48, compaction requested=false 2024-11-26T16:42:16,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:16,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:16,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-26T16:42:16,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-26T16:42:16,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-26T16:42:16,540 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8860 sec 2024-11-26T16:42:16,541 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.8930 sec 2024-11-26T16:42:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:16,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28a7a40a9a40603189977c8da6b1d6a7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T16:42:16,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=A 2024-11-26T16:42:16,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:16,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=B 2024-11-26T16:42:16,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:16,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=C 2024-11-26T16:42:16,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:16,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262750b5da718b4addaee97866b248510e_28a7a40a9a40603189977c8da6b1d6a7 is 50, key is test_row_0/A:col10/1732639335399/Put/seqid=0 2024-11-26T16:42:16,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742164_1340 (size=14594) 2024-11-26T16:42:16,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639396630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:16,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732639336599, max timestamp=1732639336640Num uniques 859; 2024-11-26T16:42:16,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T16:42:16,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:16,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732639336599, max timestamp=1732639336640Num uniques 859; 2024-11-26T16:42:16,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T16:42:16,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732639336599, max timestamp=1732639336640Num uniques 859; 2024-11-26T16:42:16,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T16:42:16,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:16,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639396693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:16,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639396694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:16,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:16,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639396709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:16,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:16,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:16,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639396751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:16,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-26T16:42:16,802 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:16,802 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:42:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-26T16:42:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T16:42:16,804 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:42:16,805 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:42:16,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:42:16,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639396820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:16,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639396821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:16,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639396831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:16,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T16:42:16,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T16:42:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:16,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:16,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639396961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:16,973 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_0' in 28a7a40a9a40603189977c8da6b1d6a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:16,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:16,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:16,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639396971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:16,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:16,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:16,996 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:17,000 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411262750b5da718b4addaee97866b248510e_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262750b5da718b4addaee97866b248510e_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:17,001 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/f728f64eaaeb4e96acfb0aac28da6beb, store: [table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:17,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/f728f64eaaeb4e96acfb0aac28da6beb is 175, key is test_row_0/A:col10/1732639335399/Put/seqid=0 2024-11-26T16:42:17,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742165_1341 (size=39549) 2024-11-26T16:42:17,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639397041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:17,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639397041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:17,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:17,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:17,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639397051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:17,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T16:42:17,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:17,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:17,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:17,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:17,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T16:42:17,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T16:42:17,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:17,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:17,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:17,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:17,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:17,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639397291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:17,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639397361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:17,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639397362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:17,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639397371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:17,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:17,407 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=59, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/f728f64eaaeb4e96acfb0aac28da6beb 2024-11-26T16:42:17,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/38c5e885e42c4ee785b5d4594216a433 is 50, key is test_row_0/B:col10/1732639335399/Put/seqid=0 2024-11-26T16:42:17,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T16:42:17,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:17,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:17,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:17,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:17,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:17,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742166_1342 (size=12001) 2024-11-26T16:42:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:17,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T16:42:17,532 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T16:42:17,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T16:42:17,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:17,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:17,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:17,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:17,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:17,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:17,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T16:42:17,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:17,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:17,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:17,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:17,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:17,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639397801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:17,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:17,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/38c5e885e42c4ee785b5d4594216a433 2024-11-26T16:42:17,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/7d37c5055ffb4c77b0223432756a8d4b is 50, key is test_row_0/C:col10/1732639335399/Put/seqid=0 2024-11-26T16:42:17,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742167_1343 (size=12001) 2024-11-26T16:42:17,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T16:42:17,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:17,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:17,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:17,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:17,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:17,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639397871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639397871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:17,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639397891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T16:42:18,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T16:42:18,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:18,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:18,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:18,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:18,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:18,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T16:42:18,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:18,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:18,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:18,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:18,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:18,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/7d37c5055ffb4c77b0223432756a8d4b 2024-11-26T16:42:18,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/f728f64eaaeb4e96acfb0aac28da6beb as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/f728f64eaaeb4e96acfb0aac28da6beb 2024-11-26T16:42:18,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/f728f64eaaeb4e96acfb0aac28da6beb, entries=200, sequenceid=59, filesize=38.6 K 2024-11-26T16:42:18,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/38c5e885e42c4ee785b5d4594216a433 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/38c5e885e42c4ee785b5d4594216a433 2024-11-26T16:42:18,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/38c5e885e42c4ee785b5d4594216a433, entries=150, sequenceid=59, filesize=11.7 K 2024-11-26T16:42:18,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/7d37c5055ffb4c77b0223432756a8d4b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/7d37c5055ffb4c77b0223432756a8d4b 2024-11-26T16:42:18,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/7d37c5055ffb4c77b0223432756a8d4b, entries=150, sequenceid=59, filesize=11.7 K 2024-11-26T16:42:18,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 28a7a40a9a40603189977c8da6b1d6a7 in 1672ms, sequenceid=59, compaction requested=true 2024-11-26T16:42:18,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:18,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:42:18,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:18,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:42:18,257 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:18,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:18,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:18,257 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:18,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:18,258 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:18,258 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:18,259 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/A is initiating minor compaction (all files) 2024-11-26T16:42:18,259 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/B is initiating minor compaction (all files) 2024-11-26T16:42:18,259 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/A in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:18,259 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/B in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:18,259 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/5be0f4eb416547f2a437b8339cd0d534, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4a0d00113f99499ca67027eb1f013f11, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/f728f64eaaeb4e96acfb0aac28da6beb] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=107.5 K 2024-11-26T16:42:18,259 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/560ccab703b14672ac4ffccc7cea038b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/41f26ffe39374165a7dc76b4aa3af301, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/38c5e885e42c4ee785b5d4594216a433] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=35.2 K 2024-11-26T16:42:18,259 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:18,259 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. files: [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/5be0f4eb416547f2a437b8339cd0d534, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4a0d00113f99499ca67027eb1f013f11, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/f728f64eaaeb4e96acfb0aac28da6beb] 2024-11-26T16:42:18,259 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 560ccab703b14672ac4ffccc7cea038b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732639334658 2024-11-26T16:42:18,259 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5be0f4eb416547f2a437b8339cd0d534, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732639334657 2024-11-26T16:42:18,259 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 41f26ffe39374165a7dc76b4aa3af301, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732639334742 2024-11-26T16:42:18,260 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a0d00113f99499ca67027eb1f013f11, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732639334742 2024-11-26T16:42:18,260 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 38c5e885e42c4ee785b5d4594216a433, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732639335399 2024-11-26T16:42:18,260 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting f728f64eaaeb4e96acfb0aac28da6beb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732639335399 2024-11-26T16:42:18,268 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:18,270 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#B#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:18,270 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126d532b0eaca6c434ea9654bb3ad3839e8_28a7a40a9a40603189977c8da6b1d6a7 store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:18,270 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/96250f39c6c842a8a26201019e69c54d is 50, key is test_row_0/B:col10/1732639335399/Put/seqid=0 2024-11-26T16:42:18,272 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126d532b0eaca6c434ea9654bb3ad3839e8_28a7a40a9a40603189977c8da6b1d6a7, store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:18,272 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d532b0eaca6c434ea9654bb3ad3839e8_28a7a40a9a40603189977c8da6b1d6a7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:18,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742168_1344 (size=12104) 2024-11-26T16:42:18,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742169_1345 (size=4469) 2024-11-26T16:42:18,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-26T16:42:18,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:18,338 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 28a7a40a9a40603189977c8da6b1d6a7 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-26T16:42:18,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=A 2024-11-26T16:42:18,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:18,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=B 2024-11-26T16:42:18,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:18,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=C 2024-11-26T16:42:18,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:18,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411266c9ecb68d8d64e0ea788de7f19e7cd28_28a7a40a9a40603189977c8da6b1d6a7 is 50, key is test_row_0/A:col10/1732639336693/Put/seqid=0 2024-11-26T16:42:18,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742170_1346 (size=12154) 2024-11-26T16:42:18,687 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/96250f39c6c842a8a26201019e69c54d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/96250f39c6c842a8a26201019e69c54d 2024-11-26T16:42:18,689 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#A#compaction#295 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:18,690 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/c740d8e0e95a4e3d8c1b7e2f80ed3cdd is 175, key is test_row_0/A:col10/1732639335399/Put/seqid=0 2024-11-26T16:42:18,692 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/B of 28a7a40a9a40603189977c8da6b1d6a7 into 96250f39c6c842a8a26201019e69c54d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:18,692 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:18,692 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/B, priority=13, startTime=1732639338257; duration=0sec 2024-11-26T16:42:18,692 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:18,693 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:B 2024-11-26T16:42:18,693 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:18,696 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:18,696 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/C is initiating minor compaction (all files) 2024-11-26T16:42:18,696 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/C in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:18,696 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/9b818ef5b36e468d8c96e028602f2ed4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/a48c36efa2454fe295386b2edfac4895, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/7d37c5055ffb4c77b0223432756a8d4b] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=35.2 K 2024-11-26T16:42:18,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742171_1347 (size=31058) 2024-11-26T16:42:18,699 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b818ef5b36e468d8c96e028602f2ed4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732639334658 2024-11-26T16:42:18,700 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a48c36efa2454fe295386b2edfac4895, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732639334742 2024-11-26T16:42:18,700 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d37c5055ffb4c77b0223432756a8d4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732639335399 2024-11-26T16:42:18,707 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#C#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:18,707 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/1bfc37afe4ca4fb2b0056204cc5b2d0d is 50, key is test_row_0/C:col10/1732639335399/Put/seqid=0 2024-11-26T16:42:18,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742172_1348 (size=12104) 2024-11-26T16:42:18,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:18,754 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411266c9ecb68d8d64e0ea788de7f19e7cd28_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266c9ecb68d8d64e0ea788de7f19e7cd28_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:18,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/078f850a7090472b8efcaab09207c111, store: [table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:18,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/078f850a7090472b8efcaab09207c111 is 175, key is test_row_0/A:col10/1732639336693/Put/seqid=0 2024-11-26T16:42:18,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742173_1349 (size=30955) 2024-11-26T16:42:18,761 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/078f850a7090472b8efcaab09207c111 2024-11-26T16:42:18,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/c74837f986c044959f9f8f252be9e7cf is 50, key is test_row_0/B:col10/1732639336693/Put/seqid=0 2024-11-26T16:42:18,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742174_1350 (size=12001) 2024-11-26T16:42:18,783 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/c74837f986c044959f9f8f252be9e7cf 2024-11-26T16:42:18,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/c85a09885667457cb3a2d83d171da24d is 50, key is test_row_0/C:col10/1732639336693/Put/seqid=0 2024-11-26T16:42:18,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742175_1351 (size=12001) 2024-11-26T16:42:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:18,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:18,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639398907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:18,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:18,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:18,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:18,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639398908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:18,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639398909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:18,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:18,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:18,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639398911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:18,924 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_0' in 28a7a40a9a40603189977c8da6b1d6a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T16:42:18,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639398991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:18,998 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_0' in 28a7a40a9a40603189977c8da6b1d6a7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:19,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639399021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:19,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639399031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:19,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639399031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:19,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:19,102 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/c740d8e0e95a4e3d8c1b7e2f80ed3cdd as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/c740d8e0e95a4e3d8c1b7e2f80ed3cdd 2024-11-26T16:42:19,106 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/A of 28a7a40a9a40603189977c8da6b1d6a7 into c740d8e0e95a4e3d8c1b7e2f80ed3cdd(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:19,106 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:19,106 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/A, priority=13, startTime=1732639338257; duration=0sec 2024-11-26T16:42:19,106 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:19,107 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:A 2024-11-26T16:42:19,117 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/1bfc37afe4ca4fb2b0056204cc5b2d0d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/1bfc37afe4ca4fb2b0056204cc5b2d0d 2024-11-26T16:42:19,122 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/C of 28a7a40a9a40603189977c8da6b1d6a7 into 1bfc37afe4ca4fb2b0056204cc5b2d0d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:19,122 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:19,122 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/C, priority=13, startTime=1732639338257; duration=0sec 2024-11-26T16:42:19,122 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:19,122 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:C 2024-11-26T16:42:19,210 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/c85a09885667457cb3a2d83d171da24d 2024-11-26T16:42:19,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/078f850a7090472b8efcaab09207c111 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/078f850a7090472b8efcaab09207c111 2024-11-26T16:42:19,218 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/078f850a7090472b8efcaab09207c111, entries=150, sequenceid=93, filesize=30.2 K 2024-11-26T16:42:19,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/c74837f986c044959f9f8f252be9e7cf as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/c74837f986c044959f9f8f252be9e7cf 2024-11-26T16:42:19,223 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/c74837f986c044959f9f8f252be9e7cf, entries=150, sequenceid=93, filesize=11.7 K 2024-11-26T16:42:19,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/c85a09885667457cb3a2d83d171da24d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/c85a09885667457cb3a2d83d171da24d 2024-11-26T16:42:19,227 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/c85a09885667457cb3a2d83d171da24d, entries=150, sequenceid=93, filesize=11.7 K 2024-11-26T16:42:19,228 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 28a7a40a9a40603189977c8da6b1d6a7 in 890ms, sequenceid=93, compaction requested=false 2024-11-26T16:42:19,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:19,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:19,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-26T16:42:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-26T16:42:19,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-26T16:42:19,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4250 sec 2024-11-26T16:42:19,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.4300 sec 2024-11-26T16:42:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:19,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28a7a40a9a40603189977c8da6b1d6a7 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T16:42:19,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=A 2024-11-26T16:42:19,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:19,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=B 2024-11-26T16:42:19,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:19,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=C 2024-11-26T16:42:19,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:19,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112654c68a8850344f20b241deffd6daecbb_28a7a40a9a40603189977c8da6b1d6a7 is 50, key is test_row_0/A:col10/1732639339242/Put/seqid=0 2024-11-26T16:42:19,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742176_1352 (size=14594) 2024-11-26T16:42:19,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639399298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:19,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:19,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639399299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:19,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:19,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639399307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:19,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:19,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639399411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:19,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:19,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639399421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:19,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639399421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:19,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:19,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639399632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:19,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:19,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639399632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639399632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:19,661 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:19,668 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112654c68a8850344f20b241deffd6daecbb_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112654c68a8850344f20b241deffd6daecbb_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:19,672 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/712758c03998454186aae6adb9fe84ae, store: [table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:19,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/712758c03998454186aae6adb9fe84ae is 175, key is test_row_0/A:col10/1732639339242/Put/seqid=0 2024-11-26T16:42:19,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742177_1353 (size=39549) 2024-11-26T16:42:19,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639399951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:19,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:19,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639399951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:19,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:19,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:19,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639399951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:19,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:19,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:20,085 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/712758c03998454186aae6adb9fe84ae 2024-11-26T16:42:20,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/27874bbe1f7c46af955dacc6787d467a is 50, key is test_row_0/B:col10/1732639339242/Put/seqid=0 2024-11-26T16:42:20,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742178_1354 (size=12001) 2024-11-26T16:42:20,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639400471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:20,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639400471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:20,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639400471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:20,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/27874bbe1f7c46af955dacc6787d467a 2024-11-26T16:42:20,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/a7789461c8324d828b4d77f4abe2477b is 50, key is test_row_0/C:col10/1732639339242/Put/seqid=0 2024-11-26T16:42:20,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742179_1355 (size=12001) 2024-11-26T16:42:20,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/a7789461c8324d828b4d77f4abe2477b 2024-11-26T16:42:20,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/712758c03998454186aae6adb9fe84ae as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/712758c03998454186aae6adb9fe84ae 2024-11-26T16:42:20,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/712758c03998454186aae6adb9fe84ae, entries=200, sequenceid=108, filesize=38.6 K 2024-11-26T16:42:20,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/27874bbe1f7c46af955dacc6787d467a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/27874bbe1f7c46af955dacc6787d467a 2024-11-26T16:42:20,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/27874bbe1f7c46af955dacc6787d467a, entries=150, sequenceid=108, filesize=11.7 K 2024-11-26T16:42:20,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/a7789461c8324d828b4d77f4abe2477b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/a7789461c8324d828b4d77f4abe2477b 2024-11-26T16:42:20,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/a7789461c8324d828b4d77f4abe2477b, entries=150, sequenceid=108, filesize=11.7 K 2024-11-26T16:42:20,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 28a7a40a9a40603189977c8da6b1d6a7 in 1693ms, sequenceid=108, compaction requested=true 2024-11-26T16:42:20,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:20,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:42:20,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:20,936 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:20,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:42:20,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:20,936 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:20,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:20,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:20,937 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:20,937 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/A is initiating minor compaction (all files) 2024-11-26T16:42:20,937 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/A in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:20,937 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/c740d8e0e95a4e3d8c1b7e2f80ed3cdd, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/078f850a7090472b8efcaab09207c111, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/712758c03998454186aae6adb9fe84ae] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=99.2 K 2024-11-26T16:42:20,938 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:20,938 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. files: [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/c740d8e0e95a4e3d8c1b7e2f80ed3cdd, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/078f850a7090472b8efcaab09207c111, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/712758c03998454186aae6adb9fe84ae] 2024-11-26T16:42:20,938 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:20,938 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/B is initiating minor compaction (all files) 2024-11-26T16:42:20,938 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/B in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:20,938 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting c740d8e0e95a4e3d8c1b7e2f80ed3cdd, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732639335399 2024-11-26T16:42:20,938 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/96250f39c6c842a8a26201019e69c54d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/c74837f986c044959f9f8f252be9e7cf, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/27874bbe1f7c46af955dacc6787d467a] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=35.3 K 2024-11-26T16:42:20,938 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 078f850a7090472b8efcaab09207c111, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732639336689 2024-11-26T16:42:20,938 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 96250f39c6c842a8a26201019e69c54d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732639335399 2024-11-26T16:42:20,939 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 712758c03998454186aae6adb9fe84ae, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732639338904 2024-11-26T16:42:20,939 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting c74837f986c044959f9f8f252be9e7cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732639336689 2024-11-26T16:42:20,939 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 27874bbe1f7c46af955dacc6787d467a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732639338904 2024-11-26T16:42:20,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639339273, max timestamp=1732639340941Num uniques 859; 2024-11-26T16:42:20,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639339273, max timestamp=1732639340941Num uniques 859; 2024-11-26T16:42:20,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:20,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:20,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639339273, max timestamp=1732639340941Num uniques 859; 2024-11-26T16:42:20,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:20,950 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:20,951 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#B#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:20,952 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/a8c7ea83aee44797b9d7edb7e013882e is 50, key is test_row_0/B:col10/1732639339242/Put/seqid=0 2024-11-26T16:42:20,952 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112633d32dd42d3f4817b53eb6439022e6c2_28a7a40a9a40603189977c8da6b1d6a7 store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:20,954 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112633d32dd42d3f4817b53eb6439022e6c2_28a7a40a9a40603189977c8da6b1d6a7, store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:20,955 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112633d32dd42d3f4817b53eb6439022e6c2_28a7a40a9a40603189977c8da6b1d6a7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:20,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742181_1357 (size=4469) 2024-11-26T16:42:20,965 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#A#compaction#304 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:20,966 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/5196fdbd647f46b398e1f4352a142fc8 is 175, key is test_row_0/A:col10/1732639339242/Put/seqid=0 2024-11-26T16:42:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-26T16:42:20,972 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:20,972 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:42:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-26T16:42:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T16:42:20,974 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:42:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:20,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28a7a40a9a40603189977c8da6b1d6a7 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-26T16:42:20,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=A 2024-11-26T16:42:20,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:20,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=B 2024-11-26T16:42:20,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:20,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=C 2024-11-26T16:42:20,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:20,975 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:42:20,976 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:42:20,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742180_1356 (size=12207) 2024-11-26T16:42:20,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742182_1358 (size=31161) 2024-11-26T16:42:20,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a456df9a2c6d48dbb56a568269050b74_28a7a40a9a40603189977c8da6b1d6a7 is 50, key is test_row_0/A:col10/1732639340973/Put/seqid=0 2024-11-26T16:42:20,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742183_1359 (size=14744) 2024-11-26T16:42:20,998 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:21,003 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a456df9a2c6d48dbb56a568269050b74_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a456df9a2c6d48dbb56a568269050b74_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:21,004 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/b77e1ae5e42542a6a61b75ba02d333e4, store: [table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:21,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/b77e1ae5e42542a6a61b75ba02d333e4 is 175, key is test_row_0/A:col10/1732639340973/Put/seqid=0 2024-11-26T16:42:21,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742184_1360 (size=39699) 2024-11-26T16:42:21,011 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/b77e1ae5e42542a6a61b75ba02d333e4 2024-11-26T16:42:21,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/a04ba914cb0f41508f60fee1d8a27afc is 50, key is test_row_0/B:col10/1732639340973/Put/seqid=0 2024-11-26T16:42:21,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742185_1361 (size=12101) 2024-11-26T16:42:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T16:42:21,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T16:42:21,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:21,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:21,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:21,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:21,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:21,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=109, cellSize=4985, totalHeapSize=11576, min timestamp=1732639341164, max timestamp=1732639341185Num uniques 859; 2024-11-26T16:42:21,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=4985, suffix heap size=117064,new segment heap size=11576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=109 2024-11-26T16:42:21,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=109, cellSize=4985, totalHeapSize=11576, min timestamp=1732639341164, max timestamp=1732639341185Num uniques 859; 2024-11-26T16:42:21,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=109, cellSize=4985, totalHeapSize=11576, min timestamp=1732639341164, max timestamp=1732639341185Num uniques 859; 2024-11-26T16:42:21,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=4985, suffix heap size=117064,new segment heap size=11576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=109 2024-11-26T16:42:21,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=4985, suffix heap size=117064,new segment heap size=11576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=109 2024-11-26T16:42:21,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T16:42:21,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:21,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:21,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:21,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:21,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:21,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:21,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T16:42:21,387 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/a8c7ea83aee44797b9d7edb7e013882e as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/a8c7ea83aee44797b9d7edb7e013882e 2024-11-26T16:42:21,388 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/5196fdbd647f46b398e1f4352a142fc8 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/5196fdbd647f46b398e1f4352a142fc8 2024-11-26T16:42:21,394 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/B of 28a7a40a9a40603189977c8da6b1d6a7 into a8c7ea83aee44797b9d7edb7e013882e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:21,394 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:21,394 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/B, priority=13, startTime=1732639340936; duration=0sec 2024-11-26T16:42:21,395 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:21,395 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:B 2024-11-26T16:42:21,395 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:21,395 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/A of 28a7a40a9a40603189977c8da6b1d6a7 into 5196fdbd647f46b398e1f4352a142fc8(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:21,395 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:21,395 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/A, priority=13, startTime=1732639340936; duration=0sec 2024-11-26T16:42:21,395 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:21,395 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:A 2024-11-26T16:42:21,398 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:21,398 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/C is initiating minor compaction (all files) 2024-11-26T16:42:21,398 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/C in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:21,398 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/1bfc37afe4ca4fb2b0056204cc5b2d0d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/c85a09885667457cb3a2d83d171da24d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/a7789461c8324d828b4d77f4abe2477b] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=35.3 K 2024-11-26T16:42:21,399 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bfc37afe4ca4fb2b0056204cc5b2d0d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732639335399 2024-11-26T16:42:21,399 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting c85a09885667457cb3a2d83d171da24d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732639336689 2024-11-26T16:42:21,399 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a7789461c8324d828b4d77f4abe2477b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732639338904 2024-11-26T16:42:21,405 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#C#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:21,405 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/c2e847be82134be3b5f9a825a66ca895 is 50, key is test_row_0/C:col10/1732639339242/Put/seqid=0 2024-11-26T16:42:21,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639401397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:21,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:21,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:21,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:21,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742186_1362 (size=12207) 2024-11-26T16:42:21,415 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/c2e847be82134be3b5f9a825a66ca895 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/c2e847be82134be3b5f9a825a66ca895 2024-11-26T16:42:21,419 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/C of 28a7a40a9a40603189977c8da6b1d6a7 into c2e847be82134be3b5f9a825a66ca895(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:21,419 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:21,419 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/C, priority=13, startTime=1732639340936; duration=0sec 2024-11-26T16:42:21,419 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:21,419 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:C 2024-11-26T16:42:21,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/a04ba914cb0f41508f60fee1d8a27afc 2024-11-26T16:42:21,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T16:42:21,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:21,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:21,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:21,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:21,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:21,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/05380e1ff3db43aea4de8d4a88f7e734 is 50, key is test_row_0/C:col10/1732639340973/Put/seqid=0 2024-11-26T16:42:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:21,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742187_1363 (size=12101) 2024-11-26T16:42:21,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:21,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639401491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:21,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:21,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639401491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:21,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:21,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639401492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:21,507 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in 28a7a40a9a40603189977c8da6b1d6a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:21,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:21,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:21,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:21,507 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in 28a7a40a9a40603189977c8da6b1d6a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:21,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:21,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:21,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:21,507 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in 28a7a40a9a40603189977c8da6b1d6a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:21,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:21,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:21,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:21,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639401511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:21,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:21,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:21,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:21,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T16:42:21,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:21,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:21,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:21,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:21,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:21,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T16:42:21,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639401731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:21,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T16:42:21,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:21,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:21,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:21,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:21,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:21,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/05380e1ff3db43aea4de8d4a88f7e734 2024-11-26T16:42:21,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/b77e1ae5e42542a6a61b75ba02d333e4 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/b77e1ae5e42542a6a61b75ba02d333e4 2024-11-26T16:42:21,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/b77e1ae5e42542a6a61b75ba02d333e4, entries=200, sequenceid=133, filesize=38.8 K 2024-11-26T16:42:21,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/a04ba914cb0f41508f60fee1d8a27afc as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/a04ba914cb0f41508f60fee1d8a27afc 2024-11-26T16:42:21,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/a04ba914cb0f41508f60fee1d8a27afc, entries=150, sequenceid=133, filesize=11.8 K 2024-11-26T16:42:21,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/05380e1ff3db43aea4de8d4a88f7e734 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/05380e1ff3db43aea4de8d4a88f7e734 2024-11-26T16:42:21,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/05380e1ff3db43aea4de8d4a88f7e734, entries=150, sequenceid=133, filesize=11.8 K 2024-11-26T16:42:21,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=127.47 KB/130530 for 28a7a40a9a40603189977c8da6b1d6a7 in 889ms, sequenceid=133, compaction requested=false 2024-11-26T16:42:21,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:21,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-26T16:42:21,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:21,895 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 28a7a40a9a40603189977c8da6b1d6a7 3/3 column families, dataSize=127.47 KB heapSize=369.75 KB 2024-11-26T16:42:21,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=A 2024-11-26T16:42:21,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:21,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=B 2024-11-26T16:42:21,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:21,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=C 2024-11-26T16:42:21,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:21,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126967976dd31f5458eae894bea313f13b0_28a7a40a9a40603189977c8da6b1d6a7 is 50, key is test_row_0/A:col10/1732639341395/Put/seqid=0 2024-11-26T16:42:21,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742188_1364 (size=12304) 2024-11-26T16:42:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:22,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T16:42:22,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639402119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:22,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639402241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:22,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:22,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:22,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:22,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:22,311 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126967976dd31f5458eae894bea313f13b0_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126967976dd31f5458eae894bea313f13b0_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:22,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/4d48e57613ca43f9bff7bb7912f5ea65, store: [table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:22,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/4d48e57613ca43f9bff7bb7912f5ea65 is 175, key is test_row_0/A:col10/1732639341395/Put/seqid=0 2024-11-26T16:42:22,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742189_1365 (size=31105) 2024-11-26T16:42:22,319 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/4d48e57613ca43f9bff7bb7912f5ea65 2024-11-26T16:42:22,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/ebf3fe17a4284e4193fbd8cb8881cf14 is 50, key is test_row_0/B:col10/1732639341395/Put/seqid=0 2024-11-26T16:42:22,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742190_1366 (size=12151) 2024-11-26T16:42:22,340 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/ebf3fe17a4284e4193fbd8cb8881cf14 2024-11-26T16:42:22,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/050888ee3d3a46bbb22709d2de63ef06 is 50, key is test_row_0/C:col10/1732639341395/Put/seqid=0 2024-11-26T16:42:22,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742191_1367 (size=12151) 2024-11-26T16:42:22,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639402451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:22,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:22,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:22,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:22,758 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/050888ee3d3a46bbb22709d2de63ef06 2024-11-26T16:42:22,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/4d48e57613ca43f9bff7bb7912f5ea65 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4d48e57613ca43f9bff7bb7912f5ea65 2024-11-26T16:42:22,777 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4d48e57613ca43f9bff7bb7912f5ea65, entries=150, sequenceid=173, filesize=30.4 K 2024-11-26T16:42:22,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/ebf3fe17a4284e4193fbd8cb8881cf14 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/ebf3fe17a4284e4193fbd8cb8881cf14 2024-11-26T16:42:22,782 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/ebf3fe17a4284e4193fbd8cb8881cf14, entries=150, sequenceid=173, filesize=11.9 K 2024-11-26T16:42:22,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/050888ee3d3a46bbb22709d2de63ef06 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/050888ee3d3a46bbb22709d2de63ef06 2024-11-26T16:42:22,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639402771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:22,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:22,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:22,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:22,790 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/050888ee3d3a46bbb22709d2de63ef06, entries=150, sequenceid=173, filesize=11.9 K 2024-11-26T16:42:22,791 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~369.70 KB/378576, currentSize=53.67 KB/54960 for 28a7a40a9a40603189977c8da6b1d6a7 in 896ms, sequenceid=173, compaction requested=true 2024-11-26T16:42:22,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:22,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:22,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-26T16:42:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-26T16:42:22,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-26T16:42:22,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8160 sec 2024-11-26T16:42:22,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8230 sec 2024-11-26T16:42:23,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28a7a40a9a40603189977c8da6b1d6a7 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T16:42:23,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=A 2024-11-26T16:42:23,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:23,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=B 2024-11-26T16:42:23,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:23,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=C 2024-11-26T16:42:23,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:23,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a5f9daea894347778cc696e114e63912_28a7a40a9a40603189977c8da6b1d6a7 is 50, key is test_row_0/A:col10/1732639343016/Put/seqid=0 2024-11-26T16:42:23,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742192_1368 (size=14794) 2024-11-26T16:42:23,067 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:23,073 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126a5f9daea894347778cc696e114e63912_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a5f9daea894347778cc696e114e63912_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:23,077 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/77ca0b544a9f44b5b89fb749b4284999, store: [table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:23,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/77ca0b544a9f44b5b89fb749b4284999 is 175, key is test_row_0/A:col10/1732639343016/Put/seqid=0 2024-11-26T16:42:23,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742193_1369 (size=39749) 2024-11-26T16:42:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-26T16:42:23,121 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:23,121 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:42:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-26T16:42:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T16:42:23,123 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:42:23,124 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:42:23,124 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:42:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T16:42:23,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639403262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:23,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-26T16:42:23,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:23,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:23,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:23,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:23,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:23,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:23,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:23,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:23,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639403301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:23,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639403381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:23,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-26T16:42:23,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:23,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:23,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:23,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:23,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T16:42:23,482 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=185, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/77ca0b544a9f44b5b89fb749b4284999 2024-11-26T16:42:23,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/b4d64309ddc347ec96b5642e1b4d0bd3 is 50, key is test_row_0/B:col10/1732639343016/Put/seqid=0 2024-11-26T16:42:23,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742194_1370 (size=12151) 2024-11-26T16:42:23,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639403521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:23,529 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in 28a7a40a9a40603189977c8da6b1d6a7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:23,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:23,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:23,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:23,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639403530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:23,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639403531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:23,540 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in 28a7a40a9a40603189977c8da6b1d6a7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:23,540 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in 28a7a40a9a40603189977c8da6b1d6a7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:23,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:23,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:23,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:23,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:23,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:23,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:23,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-26T16:42:23,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:23,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:23,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:23,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:23,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:23,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639403591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:23,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-26T16:42:23,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:23,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:23,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:23,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:23,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T16:42:23,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-26T16:42:23,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:23,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:23,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:23,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:23,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:23,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/b4d64309ddc347ec96b5642e1b4d0bd3 2024-11-26T16:42:23,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/00d0c4e3e21b4ee88dcaa1f528cb2b11 is 50, key is test_row_0/C:col10/1732639343016/Put/seqid=0 2024-11-26T16:42:23,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742195_1371 (size=12151) 2024-11-26T16:42:23,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639403911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:24,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-26T16:42:24,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:24,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:24,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:24,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:24,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:24,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-26T16:42:24,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:24,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:24,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:24,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:24,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T16:42:24,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/00d0c4e3e21b4ee88dcaa1f528cb2b11 2024-11-26T16:42:24,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/77ca0b544a9f44b5b89fb749b4284999 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/77ca0b544a9f44b5b89fb749b4284999 2024-11-26T16:42:24,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/77ca0b544a9f44b5b89fb749b4284999, entries=200, sequenceid=185, filesize=38.8 K 2024-11-26T16:42:24,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/b4d64309ddc347ec96b5642e1b4d0bd3 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b4d64309ddc347ec96b5642e1b4d0bd3 2024-11-26T16:42:24,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b4d64309ddc347ec96b5642e1b4d0bd3, entries=150, sequenceid=185, filesize=11.9 K 2024-11-26T16:42:24,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/00d0c4e3e21b4ee88dcaa1f528cb2b11 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/00d0c4e3e21b4ee88dcaa1f528cb2b11 2024-11-26T16:42:24,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/00d0c4e3e21b4ee88dcaa1f528cb2b11, entries=150, sequenceid=185, filesize=11.9 K 2024-11-26T16:42:24,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 28a7a40a9a40603189977c8da6b1d6a7 in 1311ms, sequenceid=185, compaction requested=true 2024-11-26T16:42:24,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:24,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:42:24,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:24,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:42:24,329 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:24,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:24,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:24,329 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:24,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:24,330 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141714 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:24,330 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/A is initiating minor compaction (all files) 2024-11-26T16:42:24,330 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48610 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:24,330 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/A in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:24,330 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/B is initiating minor compaction (all files) 2024-11-26T16:42:24,330 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/5196fdbd647f46b398e1f4352a142fc8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/b77e1ae5e42542a6a61b75ba02d333e4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4d48e57613ca43f9bff7bb7912f5ea65, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/77ca0b544a9f44b5b89fb749b4284999] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=138.4 K 2024-11-26T16:42:24,330 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/B in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:24,330 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:24,330 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. files: [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/5196fdbd647f46b398e1f4352a142fc8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/b77e1ae5e42542a6a61b75ba02d333e4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4d48e57613ca43f9bff7bb7912f5ea65, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/77ca0b544a9f44b5b89fb749b4284999] 2024-11-26T16:42:24,330 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/a8c7ea83aee44797b9d7edb7e013882e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/a04ba914cb0f41508f60fee1d8a27afc, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/ebf3fe17a4284e4193fbd8cb8881cf14, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b4d64309ddc347ec96b5642e1b4d0bd3] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=47.5 K 2024-11-26T16:42:24,330 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5196fdbd647f46b398e1f4352a142fc8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732639338904 2024-11-26T16:42:24,330 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a8c7ea83aee44797b9d7edb7e013882e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732639338904 2024-11-26T16:42:24,331 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting b77e1ae5e42542a6a61b75ba02d333e4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732639339289 2024-11-26T16:42:24,331 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a04ba914cb0f41508f60fee1d8a27afc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732639339289 2024-11-26T16:42:24,331 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting ebf3fe17a4284e4193fbd8cb8881cf14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732639341353 2024-11-26T16:42:24,331 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d48e57613ca43f9bff7bb7912f5ea65, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732639341353 2024-11-26T16:42:24,331 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77ca0b544a9f44b5b89fb749b4284999, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732639342070 2024-11-26T16:42:24,331 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting b4d64309ddc347ec96b5642e1b4d0bd3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732639342096 2024-11-26T16:42:24,338 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:24,341 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#B#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:24,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28a7a40a9a40603189977c8da6b1d6a7 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-26T16:42:24,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=A 2024-11-26T16:42:24,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:24,342 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/f308a78d697a49f79a98efad5c123b6a is 50, key is test_row_0/B:col10/1732639343016/Put/seqid=0 2024-11-26T16:42:24,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=B 2024-11-26T16:42:24,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:24,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=C 2024-11-26T16:42:24,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:24,345 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126c111b25867f64d38b7dc96c25ff5049e_28a7a40a9a40603189977c8da6b1d6a7 store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:24,348 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126c111b25867f64d38b7dc96c25ff5049e_28a7a40a9a40603189977c8da6b1d6a7, store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:24,348 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126c111b25867f64d38b7dc96c25ff5049e_28a7a40a9a40603189977c8da6b1d6a7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:24,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-26T16:42:24,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:24,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:24,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:24,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:24,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:24,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742196_1372 (size=12493) 2024-11-26T16:42:24,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126bb65fc19e1f349968125a26786e489cf_28a7a40a9a40603189977c8da6b1d6a7 is 50, key is test_row_0/A:col10/1732639343158/Put/seqid=0 2024-11-26T16:42:24,358 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/f308a78d697a49f79a98efad5c123b6a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/f308a78d697a49f79a98efad5c123b6a 2024-11-26T16:42:24,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742197_1373 (size=4469) 2024-11-26T16:42:24,363 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#A#compaction#315 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:24,364 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/68a655ada1b743368096a99afea47e52 is 175, key is test_row_0/A:col10/1732639343016/Put/seqid=0 2024-11-26T16:42:24,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742198_1374 (size=14794) 2024-11-26T16:42:24,368 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/B of 28a7a40a9a40603189977c8da6b1d6a7 into f308a78d697a49f79a98efad5c123b6a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:24,368 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:24,368 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/B, priority=12, startTime=1732639344329; duration=0sec 2024-11-26T16:42:24,369 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:24,369 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:B 2024-11-26T16:42:24,369 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:24,371 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48610 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:24,371 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/C is initiating minor compaction (all files) 2024-11-26T16:42:24,371 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/C in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:24,371 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/c2e847be82134be3b5f9a825a66ca895, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/05380e1ff3db43aea4de8d4a88f7e734, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/050888ee3d3a46bbb22709d2de63ef06, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/00d0c4e3e21b4ee88dcaa1f528cb2b11] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=47.5 K 2024-11-26T16:42:24,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742199_1375 (size=31447) 2024-11-26T16:42:24,372 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting c2e847be82134be3b5f9a825a66ca895, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732639338904 2024-11-26T16:42:24,372 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 05380e1ff3db43aea4de8d4a88f7e734, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732639339289 2024-11-26T16:42:24,373 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 050888ee3d3a46bbb22709d2de63ef06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732639341353 2024-11-26T16:42:24,373 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 00d0c4e3e21b4ee88dcaa1f528cb2b11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732639342096 2024-11-26T16:42:24,381 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#C#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:24,381 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/6f8c41c23fef4985b8c18a49e465eb9d is 50, key is test_row_0/C:col10/1732639343016/Put/seqid=0 2024-11-26T16:42:24,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742200_1376 (size=12493) 2024-11-26T16:42:24,396 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/6f8c41c23fef4985b8c18a49e465eb9d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/6f8c41c23fef4985b8c18a49e465eb9d 2024-11-26T16:42:24,400 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/C of 28a7a40a9a40603189977c8da6b1d6a7 into 6f8c41c23fef4985b8c18a49e465eb9d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:24,400 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:24,401 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/C, priority=12, startTime=1732639344329; duration=0sec 2024-11-26T16:42:24,401 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:24,401 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:C 2024-11-26T16:42:24,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639404431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:24,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:24,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:24,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:24,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639404440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:24,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:24,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-26T16:42:24,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:24,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:24,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:24,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:24,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:24,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639404561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:24,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:24,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:24,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:24,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-26T16:42:24,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:24,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:24,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:24,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:24,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:24,766 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:24,769 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126bb65fc19e1f349968125a26786e489cf_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bb65fc19e1f349968125a26786e489cf_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:24,770 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/da1776a7f24d47f79ca5c85940ea80cd, store: [table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:24,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/da1776a7f24d47f79ca5c85940ea80cd is 175, key is test_row_0/A:col10/1732639343158/Put/seqid=0 2024-11-26T16:42:24,775 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/68a655ada1b743368096a99afea47e52 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/68a655ada1b743368096a99afea47e52 2024-11-26T16:42:24,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742201_1377 (size=39749) 2024-11-26T16:42:24,781 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/A of 28a7a40a9a40603189977c8da6b1d6a7 into 68a655ada1b743368096a99afea47e52(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:24,781 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:24,781 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/A, priority=12, startTime=1732639344328; duration=0sec 2024-11-26T16:42:24,781 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:24,781 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:A 2024-11-26T16:42:24,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639404781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:24,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:24,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-26T16:42:24,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:24,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:24,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:24,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:24,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:24,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-26T16:42:24,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:24,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:24,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:24,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:24,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:25,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639405101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:25,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:25,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:25,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:25,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-26T16:42:25,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:25,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:25,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:25,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:25,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:25,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:25,179 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/da1776a7f24d47f79ca5c85940ea80cd 2024-11-26T16:42:25,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/2e12a550bd1f47568537b58266e29fd4 is 50, key is test_row_0/B:col10/1732639343158/Put/seqid=0 2024-11-26T16:42:25,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742202_1378 (size=12151) 2024-11-26T16:42:25,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/2e12a550bd1f47568537b58266e29fd4 2024-11-26T16:42:25,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/e1124c7ffabe4b0e930d50cb34c4ec63 is 50, key is test_row_0/C:col10/1732639343158/Put/seqid=0 2024-11-26T16:42:25,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742203_1379 (size=12151) 2024-11-26T16:42:25,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-26T16:42:25,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:25,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:25,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:25,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:25,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:25,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:25,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T16:42:25,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-26T16:42:25,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:25,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:25,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:25,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:25,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:25,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639405451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:25,457 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in 28a7a40a9a40603189977c8da6b1d6a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:25,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-26T16:42:25,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:25,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:25,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:25,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:25,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:25,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/e1124c7ffabe4b0e930d50cb34c4ec63 2024-11-26T16:42:25,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/da1776a7f24d47f79ca5c85940ea80cd as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/da1776a7f24d47f79ca5c85940ea80cd 2024-11-26T16:42:25,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/da1776a7f24d47f79ca5c85940ea80cd, entries=200, sequenceid=206, filesize=38.8 K 2024-11-26T16:42:25,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/2e12a550bd1f47568537b58266e29fd4 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/2e12a550bd1f47568537b58266e29fd4 2024-11-26T16:42:25,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/2e12a550bd1f47568537b58266e29fd4, entries=150, sequenceid=206, filesize=11.9 K 2024-11-26T16:42:25,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/e1124c7ffabe4b0e930d50cb34c4ec63 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/e1124c7ffabe4b0e930d50cb34c4ec63 2024-11-26T16:42:25,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639405611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:25,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:25,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/e1124c7ffabe4b0e930d50cb34c4ec63, entries=150, sequenceid=206, filesize=11.9 K 2024-11-26T16:42:25,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:25,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 28a7a40a9a40603189977c8da6b1d6a7 in 1284ms, sequenceid=206, compaction requested=false 2024-11-26T16:42:25,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:25,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-26T16:42:25,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:25,727 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 28a7a40a9a40603189977c8da6b1d6a7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T16:42:25,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=A 2024-11-26T16:42:25,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:25,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=B 2024-11-26T16:42:25,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:25,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=C 2024-11-26T16:42:25,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:25,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112641ec1e8c16534aaa8b51428ac4189e69_28a7a40a9a40603189977c8da6b1d6a7 is 50, key is test_row_0/A:col10/1732639344420/Put/seqid=0 2024-11-26T16:42:25,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742204_1380 (size=12304) 2024-11-26T16:42:26,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:26,148 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112641ec1e8c16534aaa8b51428ac4189e69_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112641ec1e8c16534aaa8b51428ac4189e69_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:26,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/d2dbdbe08d8d4d3a931e559adab119db, store: [table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:26,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/d2dbdbe08d8d4d3a931e559adab119db is 175, key is test_row_0/A:col10/1732639344420/Put/seqid=0 2024-11-26T16:42:26,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742205_1381 (size=31105) 2024-11-26T16:42:26,554 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/d2dbdbe08d8d4d3a931e559adab119db 2024-11-26T16:42:26,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/b545789149364af88d107b5bf28c0f2c is 50, key is test_row_0/B:col10/1732639344420/Put/seqid=0 2024-11-26T16:42:26,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742206_1382 (size=12151) 2024-11-26T16:42:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:26,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:26,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639346742, max timestamp=1732639346795Num uniques 859; 2024-11-26T16:42:26,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639346742, max timestamp=1732639346795Num uniques 859; 2024-11-26T16:42:26,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:26,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:26,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639406922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:26,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:26,966 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/b545789149364af88d107b5bf28c0f2c 2024-11-26T16:42:26,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/3c10a0311d28449e8bc549e3e1d35537 is 50, key is test_row_0/C:col10/1732639344420/Put/seqid=0 2024-11-26T16:42:26,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742207_1383 (size=12151) 2024-11-26T16:42:27,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639407041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:27,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639407261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:27,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:27,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:27,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T16:42:27,377 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/3c10a0311d28449e8bc549e3e1d35537 2024-11-26T16:42:27,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/d2dbdbe08d8d4d3a931e559adab119db as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d2dbdbe08d8d4d3a931e559adab119db 2024-11-26T16:42:27,385 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d2dbdbe08d8d4d3a931e559adab119db, entries=150, sequenceid=220, filesize=30.4 K 2024-11-26T16:42:27,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/b545789149364af88d107b5bf28c0f2c as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b545789149364af88d107b5bf28c0f2c 2024-11-26T16:42:27,390 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b545789149364af88d107b5bf28c0f2c, entries=150, sequenceid=220, filesize=11.9 K 2024-11-26T16:42:27,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/3c10a0311d28449e8bc549e3e1d35537 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/3c10a0311d28449e8bc549e3e1d35537 2024-11-26T16:42:27,394 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/3c10a0311d28449e8bc549e3e1d35537, entries=150, sequenceid=220, filesize=11.9 K 2024-11-26T16:42:27,395 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 28a7a40a9a40603189977c8da6b1d6a7 in 1667ms, sequenceid=220, compaction requested=true 2024-11-26T16:42:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-26T16:42:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-26T16:42:27,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-26T16:42:27,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.2720 sec 2024-11-26T16:42:27,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 4.2780 sec 2024-11-26T16:42:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:27,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28a7a40a9a40603189977c8da6b1d6a7 3/3 column families, dataSize=131.94 KB heapSize=383.31 KB 2024-11-26T16:42:27,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=A 2024-11-26T16:42:27,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:27,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=B 2024-11-26T16:42:27,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:27,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=C 2024-11-26T16:42:27,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:27,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112627f359b915414e87a4635fc37ed73e81_28a7a40a9a40603189977c8da6b1d6a7 is 50, key is test_row_0/A:col10/1732639346921/Put/seqid=0 2024-11-26T16:42:27,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742208_1384 (size=14794) 2024-11-26T16:42:27,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54558 deadline: 1732639407561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:27,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639407564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:27,568 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in 28a7a40a9a40603189977c8da6b1d6a7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8269 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:27,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1732639407571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:27,584 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in 28a7a40a9a40603189977c8da6b1d6a7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8285 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:27,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:27,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:27,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:27,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732639407581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:27,594 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in 28a7a40a9a40603189977c8da6b1d6a7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:27,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639407581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:27,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:27,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639407671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:27,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:27,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:27,893 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:27,897 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112627f359b915414e87a4635fc37ed73e81_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112627f359b915414e87a4635fc37ed73e81_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:27,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639407891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:27,898 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/3499529ef1964ec3aaf948695689915f, store: [table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:27,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/3499529ef1964ec3aaf948695689915f is 175, key is test_row_0/A:col10/1732639346921/Put/seqid=0 2024-11-26T16:42:27,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:27,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:27,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:27,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742209_1385 (size=39749) 2024-11-26T16:42:28,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639408101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:28,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:28,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:28,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:28,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639408210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:28,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:28,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:28,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:28,309 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/3499529ef1964ec3aaf948695689915f 2024-11-26T16:42:28,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/83a9fc5d72d7417885e9f5c5a32741f7 is 50, key is test_row_0/B:col10/1732639346921/Put/seqid=0 2024-11-26T16:42:28,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742210_1386 (size=12151) 2024-11-26T16:42:28,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/83a9fc5d72d7417885e9f5c5a32741f7 2024-11-26T16:42:28,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/576710fa56f940c59175c8393c452b69 is 50, key is test_row_0/C:col10/1732639346921/Put/seqid=0 2024-11-26T16:42:28,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742211_1387 (size=12151) 2024-11-26T16:42:28,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639408731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:28,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:28,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:28,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:29,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639409121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:29,132 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in 28a7a40a9a40603189977c8da6b1d6a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:29,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:29,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:29,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:29,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/576710fa56f940c59175c8393c452b69 2024-11-26T16:42:29,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/3499529ef1964ec3aaf948695689915f as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/3499529ef1964ec3aaf948695689915f 2024-11-26T16:42:29,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/3499529ef1964ec3aaf948695689915f, entries=200, sequenceid=252, filesize=38.8 K 2024-11-26T16:42:29,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/83a9fc5d72d7417885e9f5c5a32741f7 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/83a9fc5d72d7417885e9f5c5a32741f7 2024-11-26T16:42:29,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/83a9fc5d72d7417885e9f5c5a32741f7, entries=150, sequenceid=252, filesize=11.9 K 2024-11-26T16:42:29,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/576710fa56f940c59175c8393c452b69 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/576710fa56f940c59175c8393c452b69 2024-11-26T16:42:29,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/576710fa56f940c59175c8393c452b69, entries=150, sequenceid=252, filesize=11.9 K 2024-11-26T16:42:29,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~383.27 KB/392464, currentSize=46.96 KB/48090 for 28a7a40a9a40603189977c8da6b1d6a7 in 1677ms, sequenceid=252, compaction requested=true 2024-11-26T16:42:29,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:29,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:42:29,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:29,154 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:29,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:42:29,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:29,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:29,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:29,154 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:29,155 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142050 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:29,155 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:29,155 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/A is initiating minor compaction (all files) 2024-11-26T16:42:29,155 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/B is initiating minor compaction (all files) 2024-11-26T16:42:29,155 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/A in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:29,155 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/B in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:29,155 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/68a655ada1b743368096a99afea47e52, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/da1776a7f24d47f79ca5c85940ea80cd, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d2dbdbe08d8d4d3a931e559adab119db, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/3499529ef1964ec3aaf948695689915f] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=138.7 K 2024-11-26T16:42:29,155 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/f308a78d697a49f79a98efad5c123b6a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/2e12a550bd1f47568537b58266e29fd4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b545789149364af88d107b5bf28c0f2c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/83a9fc5d72d7417885e9f5c5a32741f7] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=47.8 K 2024-11-26T16:42:29,155 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:29,155 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. files: [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/68a655ada1b743368096a99afea47e52, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/da1776a7f24d47f79ca5c85940ea80cd, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d2dbdbe08d8d4d3a931e559adab119db, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/3499529ef1964ec3aaf948695689915f] 2024-11-26T16:42:29,155 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting f308a78d697a49f79a98efad5c123b6a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732639342096 2024-11-26T16:42:29,155 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68a655ada1b743368096a99afea47e52, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732639342096 2024-11-26T16:42:29,156 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e12a550bd1f47568537b58266e29fd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732639343158 2024-11-26T16:42:29,156 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting da1776a7f24d47f79ca5c85940ea80cd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732639343158 2024-11-26T16:42:29,156 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting b545789149364af88d107b5bf28c0f2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732639344407 2024-11-26T16:42:29,156 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2dbdbe08d8d4d3a931e559adab119db, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732639344407 2024-11-26T16:42:29,156 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 83a9fc5d72d7417885e9f5c5a32741f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732639346795 2024-11-26T16:42:29,156 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3499529ef1964ec3aaf948695689915f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732639346795 2024-11-26T16:42:29,170 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:29,174 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#B#compaction#328 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:29,176 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/718a4102635641ba82511174ce1d7770 is 50, key is test_row_0/B:col10/1732639346921/Put/seqid=0 2024-11-26T16:42:29,177 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126fae3e834451b410c8e988ec51c6d9abd_28a7a40a9a40603189977c8da6b1d6a7 store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:29,181 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126fae3e834451b410c8e988ec51c6d9abd_28a7a40a9a40603189977c8da6b1d6a7, store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:29,181 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126fae3e834451b410c8e988ec51c6d9abd_28a7a40a9a40603189977c8da6b1d6a7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:29,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742212_1388 (size=12629) 2024-11-26T16:42:29,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742213_1389 (size=4469) 2024-11-26T16:42:29,191 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#A#compaction#327 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:29,191 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/de5b8fab077d466d88ccfbbcc53a19cf is 175, key is test_row_0/A:col10/1732639346921/Put/seqid=0 2024-11-26T16:42:29,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742214_1390 (size=31583) 2024-11-26T16:42:29,201 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/de5b8fab077d466d88ccfbbcc53a19cf as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/de5b8fab077d466d88ccfbbcc53a19cf 2024-11-26T16:42:29,205 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/A of 28a7a40a9a40603189977c8da6b1d6a7 into de5b8fab077d466d88ccfbbcc53a19cf(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:29,205 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:29,205 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/A, priority=12, startTime=1732639349153; duration=0sec 2024-11-26T16:42:29,205 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:29,205 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:A 2024-11-26T16:42:29,205 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:29,207 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:29,207 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/C is initiating minor compaction (all files) 2024-11-26T16:42:29,207 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/C in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:29,207 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/6f8c41c23fef4985b8c18a49e465eb9d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/e1124c7ffabe4b0e930d50cb34c4ec63, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/3c10a0311d28449e8bc549e3e1d35537, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/576710fa56f940c59175c8393c452b69] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=47.8 K 2024-11-26T16:42:29,208 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f8c41c23fef4985b8c18a49e465eb9d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732639342096 2024-11-26T16:42:29,208 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1124c7ffabe4b0e930d50cb34c4ec63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732639343158 2024-11-26T16:42:29,209 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c10a0311d28449e8bc549e3e1d35537, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732639344407 2024-11-26T16:42:29,209 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 576710fa56f940c59175c8393c452b69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732639346795 2024-11-26T16:42:29,217 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#C#compaction#329 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:29,217 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/11ac9cfb534b4d75a78bee76d84b0989 is 50, key is test_row_0/C:col10/1732639346921/Put/seqid=0 2024-11-26T16:42:29,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742215_1391 (size=12629) 2024-11-26T16:42:29,592 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/718a4102635641ba82511174ce1d7770 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/718a4102635641ba82511174ce1d7770 2024-11-26T16:42:29,596 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/B of 28a7a40a9a40603189977c8da6b1d6a7 into 718a4102635641ba82511174ce1d7770(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:29,596 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:29,596 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/B, priority=12, startTime=1732639349154; duration=0sec 2024-11-26T16:42:29,597 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:29,597 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:B 2024-11-26T16:42:29,627 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/11ac9cfb534b4d75a78bee76d84b0989 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/11ac9cfb534b4d75a78bee76d84b0989 2024-11-26T16:42:29,632 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/C of 28a7a40a9a40603189977c8da6b1d6a7 into 11ac9cfb534b4d75a78bee76d84b0989(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:29,633 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:29,633 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/C, priority=12, startTime=1732639349154; duration=0sec 2024-11-26T16:42:29,633 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:29,633 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:C 2024-11-26T16:42:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:29,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28a7a40a9a40603189977c8da6b1d6a7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T16:42:29,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=A 2024-11-26T16:42:29,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:29,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=B 2024-11-26T16:42:29,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:29,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=C 2024-11-26T16:42:29,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:29,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ead47d37c3624b47a01bf2b615249717_28a7a40a9a40603189977c8da6b1d6a7 is 50, key is test_row_0/A:col10/1732639347531/Put/seqid=0 2024-11-26T16:42:29,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742216_1392 (size=14994) 2024-11-26T16:42:29,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639349870, max timestamp=1732639349898Num uniques 859; 2024-11-26T16:42:29,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639349870, max timestamp=1732639349898Num uniques 859; 2024-11-26T16:42:29,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:29,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:29,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639349870, max timestamp=1732639349898Num uniques 859; 2024-11-26T16:42:29,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:30,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:30,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639410064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:30,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:30,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:30,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:30,177 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:30,181 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126ead47d37c3624b47a01bf2b615249717_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ead47d37c3624b47a01bf2b615249717_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:30,182 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/ad20325edc22408182b6b244339903b1, store: [table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:30,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/ad20325edc22408182b6b244339903b1 is 175, key is test_row_0/A:col10/1732639347531/Put/seqid=0 2024-11-26T16:42:30,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742217_1393 (size=39949) 2024-11-26T16:42:30,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:30,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639410182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:30,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:30,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:30,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:30,203 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/ad20325edc22408182b6b244339903b1 2024-11-26T16:42:30,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/0acdf0149ef4446bb5156fb893efcf70 is 50, key is test_row_0/B:col10/1732639347531/Put/seqid=0 2024-11-26T16:42:30,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742218_1394 (size=12301) 2024-11-26T16:42:30,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639410401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:30,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:30,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:30,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:30,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/0acdf0149ef4446bb5156fb893efcf70 2024-11-26T16:42:30,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/957bf89f20dd4cdeb75b2a71e2e26da2 is 50, key is test_row_0/C:col10/1732639347531/Put/seqid=0 2024-11-26T16:42:30,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742219_1395 (size=12301) 2024-11-26T16:42:30,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639410721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:31,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/957bf89f20dd4cdeb75b2a71e2e26da2 2024-11-26T16:42:31,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/ad20325edc22408182b6b244339903b1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/ad20325edc22408182b6b244339903b1 2024-11-26T16:42:31,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/ad20325edc22408182b6b244339903b1, entries=200, sequenceid=266, filesize=39.0 K 2024-11-26T16:42:31,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/0acdf0149ef4446bb5156fb893efcf70 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/0acdf0149ef4446bb5156fb893efcf70 2024-11-26T16:42:31,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/0acdf0149ef4446bb5156fb893efcf70, entries=150, sequenceid=266, filesize=12.0 K 2024-11-26T16:42:31,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/957bf89f20dd4cdeb75b2a71e2e26da2 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/957bf89f20dd4cdeb75b2a71e2e26da2 2024-11-26T16:42:31,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/957bf89f20dd4cdeb75b2a71e2e26da2, entries=150, sequenceid=266, filesize=12.0 K 2024-11-26T16:42:31,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 28a7a40a9a40603189977c8da6b1d6a7 in 1295ms, sequenceid=266, compaction requested=false 2024-11-26T16:42:31,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:31,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28a7a40a9a40603189977c8da6b1d6a7 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-26T16:42:31,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=A 2024-11-26T16:42:31,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:31,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=B 2024-11-26T16:42:31,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:31,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=C 2024-11-26T16:42:31,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:31,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260389a7be605440089f3a90598c4de0a0_28a7a40a9a40603189977c8da6b1d6a7 is 50, key is test_row_0/A:col10/1732639350062/Put/seqid=0 2024-11-26T16:42:31,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742220_1396 (size=14994) 2024-11-26T16:42:31,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639411229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:31,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639411241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:31,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:31,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:31,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-26T16:42:31,311 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:31,312 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:42:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-26T16:42:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T16:42:31,314 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:42:31,314 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:42:31,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:42:31,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639411351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T16:42:31,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T16:42:31,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:31,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:31,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:31,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:31,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:31,574 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:31,577 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411260389a7be605440089f3a90598c4de0a0_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260389a7be605440089f3a90598c4de0a0_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:31,578 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/8d092d07f1db4756978d659ac61b1ee9, store: [table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:31,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/8d092d07f1db4756978d659ac61b1ee9 is 175, key is test_row_0/A:col10/1732639350062/Put/seqid=0 2024-11-26T16:42:31,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:31,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639411571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:31,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:31,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:31,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:31,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742221_1397 (size=39949) 2024-11-26T16:42:31,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T16:42:31,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:31,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:31,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:31,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:31,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:31,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T16:42:31,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T16:42:31,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:31,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:31,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:31,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:31,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:31,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:31,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639411891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:31,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:31,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:31,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:31,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T16:42:31,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:31,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:31,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:31,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:31,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:31,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:31,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T16:42:31,985 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/8d092d07f1db4756978d659ac61b1ee9 2024-11-26T16:42:31,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/592d1fd305b7406ab1eb6cce60d20f4a is 50, key is test_row_0/B:col10/1732639350062/Put/seqid=0 2024-11-26T16:42:31,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742222_1398 (size=12301) 2024-11-26T16:42:32,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T16:42:32,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:32,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:32,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:32,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:32,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:32,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T16:42:32,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:32,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:32,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:32,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:32,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:32,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:32,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:32,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639412261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:32,272 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in 28a7a40a9a40603189977c8da6b1d6a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:32,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T16:42:32,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:32,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:32,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:32,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:32,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:32,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/592d1fd305b7406ab1eb6cce60d20f4a 2024-11-26T16:42:32,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/1a97f16698834b6db195e9eb9e0691e7 is 50, key is test_row_0/C:col10/1732639350062/Put/seqid=0 2024-11-26T16:42:32,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742223_1399 (size=12301) 2024-11-26T16:42:32,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639412411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:32,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:32,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:32,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:32,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T16:42:32,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T16:42:32,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:32,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:32,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:32,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:32,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:32,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:32,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T16:42:32,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:32,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:32,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:32,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:32,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:32,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/1a97f16698834b6db195e9eb9e0691e7 2024-11-26T16:42:32,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/8d092d07f1db4756978d659ac61b1ee9 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/8d092d07f1db4756978d659ac61b1ee9 2024-11-26T16:42:32,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/8d092d07f1db4756978d659ac61b1ee9, entries=200, sequenceid=303, filesize=39.0 K 2024-11-26T16:42:32,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/592d1fd305b7406ab1eb6cce60d20f4a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/592d1fd305b7406ab1eb6cce60d20f4a 2024-11-26T16:42:32,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/592d1fd305b7406ab1eb6cce60d20f4a, entries=150, sequenceid=303, filesize=12.0 K 2024-11-26T16:42:32,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/1a97f16698834b6db195e9eb9e0691e7 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/1a97f16698834b6db195e9eb9e0691e7 2024-11-26T16:42:32,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/1a97f16698834b6db195e9eb9e0691e7, entries=150, sequenceid=303, filesize=12.0 K 2024-11-26T16:42:32,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 28a7a40a9a40603189977c8da6b1d6a7 in 1667ms, sequenceid=303, compaction requested=true 2024-11-26T16:42:32,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:32,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:42:32,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:32,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:42:32,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:32,829 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:32,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:32,829 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:32,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:32,829 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:32,829 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:32,830 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/A is initiating minor compaction (all files) 2024-11-26T16:42:32,830 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/B is initiating minor compaction (all files) 2024-11-26T16:42:32,830 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/B in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:32,830 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/A in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:32,830 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/718a4102635641ba82511174ce1d7770, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/0acdf0149ef4446bb5156fb893efcf70, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/592d1fd305b7406ab1eb6cce60d20f4a] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=36.4 K 2024-11-26T16:42:32,830 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/de5b8fab077d466d88ccfbbcc53a19cf, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/ad20325edc22408182b6b244339903b1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/8d092d07f1db4756978d659ac61b1ee9] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=108.9 K 2024-11-26T16:42:32,830 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:32,830 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. files: [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/de5b8fab077d466d88ccfbbcc53a19cf, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/ad20325edc22408182b6b244339903b1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/8d092d07f1db4756978d659ac61b1ee9] 2024-11-26T16:42:32,830 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 718a4102635641ba82511174ce1d7770, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732639346795 2024-11-26T16:42:32,830 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 0acdf0149ef4446bb5156fb893efcf70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732639347531 2024-11-26T16:42:32,830 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting de5b8fab077d466d88ccfbbcc53a19cf, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732639346795 2024-11-26T16:42:32,833 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 592d1fd305b7406ab1eb6cce60d20f4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732639350045 2024-11-26T16:42:32,833 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad20325edc22408182b6b244339903b1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732639347531 2024-11-26T16:42:32,833 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d092d07f1db4756978d659ac61b1ee9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732639350030 2024-11-26T16:42:32,839 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:32,841 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#B#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:32,841 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/b0f2cedd2e0c4aa59fe6834d14113c20 is 50, key is test_row_0/B:col10/1732639350062/Put/seqid=0 2024-11-26T16:42:32,841 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126d5381ffc213f44408e9fd220b8301457_28a7a40a9a40603189977c8da6b1d6a7 store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:32,844 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126d5381ffc213f44408e9fd220b8301457_28a7a40a9a40603189977c8da6b1d6a7, store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:32,844 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d5381ffc213f44408e9fd220b8301457_28a7a40a9a40603189977c8da6b1d6a7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:32,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-26T16:42:32,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:32,846 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 28a7a40a9a40603189977c8da6b1d6a7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T16:42:32,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=A 2024-11-26T16:42:32,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:32,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=B 2024-11-26T16:42:32,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:32,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=C 2024-11-26T16:42:32,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:32,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742224_1400 (size=12881) 2024-11-26T16:42:32,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742225_1401 (size=4469) 2024-11-26T16:42:32,853 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/b0f2cedd2e0c4aa59fe6834d14113c20 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b0f2cedd2e0c4aa59fe6834d14113c20 2024-11-26T16:42:32,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264f1dbc521ef5412fb91cddb1473e9e86_28a7a40a9a40603189977c8da6b1d6a7 is 50, key is test_row_0/A:col10/1732639351227/Put/seqid=0 2024-11-26T16:42:32,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742226_1402 (size=12454) 2024-11-26T16:42:32,861 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/B of 28a7a40a9a40603189977c8da6b1d6a7 into b0f2cedd2e0c4aa59fe6834d14113c20(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:32,861 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:32,861 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/B, priority=13, startTime=1732639352828; duration=0sec 2024-11-26T16:42:32,861 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:32,862 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:B 2024-11-26T16:42:32,862 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:32,862 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:32,863 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/C is initiating minor compaction (all files) 2024-11-26T16:42:32,863 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/C in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:32,863 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/11ac9cfb534b4d75a78bee76d84b0989, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/957bf89f20dd4cdeb75b2a71e2e26da2, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/1a97f16698834b6db195e9eb9e0691e7] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=36.4 K 2024-11-26T16:42:32,863 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 11ac9cfb534b4d75a78bee76d84b0989, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732639346795 2024-11-26T16:42:32,863 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 957bf89f20dd4cdeb75b2a71e2e26da2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732639347531 2024-11-26T16:42:32,863 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a97f16698834b6db195e9eb9e0691e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732639350045 2024-11-26T16:42:32,873 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#C#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:32,873 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/6245aa7dd0ce4b84a701e0b600692727 is 50, key is test_row_0/C:col10/1732639350062/Put/seqid=0 2024-11-26T16:42:32,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742227_1403 (size=12881) 2024-11-26T16:42:32,890 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/6245aa7dd0ce4b84a701e0b600692727 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/6245aa7dd0ce4b84a701e0b600692727 2024-11-26T16:42:32,897 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/C of 28a7a40a9a40603189977c8da6b1d6a7 into 6245aa7dd0ce4b84a701e0b600692727(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:32,897 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:32,897 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/C, priority=13, startTime=1732639352829; duration=0sec 2024-11-26T16:42:32,897 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:32,897 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:C 2024-11-26T16:42:33,252 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#A#compaction#337 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:33,253 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/2d4b16b52bba41858a04ca5994a82224 is 175, key is test_row_0/A:col10/1732639350062/Put/seqid=0 2024-11-26T16:42:33,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,271 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411264f1dbc521ef5412fb91cddb1473e9e86_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264f1dbc521ef5412fb91cddb1473e9e86_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:33,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/4f61da90c5164c678a7cf1e12bffa2e8, store: [table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:33,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/4f61da90c5164c678a7cf1e12bffa2e8 is 175, key is test_row_0/A:col10/1732639351227/Put/seqid=0 2024-11-26T16:42:33,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742228_1404 (size=31835) 2024-11-26T16:42:33,285 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/2d4b16b52bba41858a04ca5994a82224 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/2d4b16b52bba41858a04ca5994a82224 2024-11-26T16:42:33,295 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/A of 28a7a40a9a40603189977c8da6b1d6a7 into 2d4b16b52bba41858a04ca5994a82224(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:33,295 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:33,295 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/A, priority=13, startTime=1732639352828; duration=0sec 2024-11-26T16:42:33,295 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:33,295 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:A 2024-11-26T16:42:33,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742229_1405 (size=31255) 2024-11-26T16:42:33,297 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/4f61da90c5164c678a7cf1e12bffa2e8 2024-11-26T16:42:33,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/21c9e12e45374566aef0ec6d5042227e is 50, key is test_row_0/B:col10/1732639351227/Put/seqid=0 2024-11-26T16:42:33,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742230_1406 (size=12301) 2024-11-26T16:42:33,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:33,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. as already flushing 2024-11-26T16:42:33,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T16:42:33,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639353486, max timestamp=1732639353624Num uniques 859; 2024-11-26T16:42:33,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:33,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639353486, max timestamp=1732639353624Num uniques 859; 2024-11-26T16:42:33,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:33,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639353486, max timestamp=1732639353624Num uniques 859; 2024-11-26T16:42:33,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:33,748 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/21c9e12e45374566aef0ec6d5042227e 2024-11-26T16:42:33,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/7ac6d12a66394010a4998cb7009318b3 is 50, key is test_row_0/C:col10/1732639351227/Put/seqid=0 2024-11-26T16:42:33,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742231_1407 (size=12301) 2024-11-26T16:42:33,773 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/7ac6d12a66394010a4998cb7009318b3 2024-11-26T16:42:33,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/4f61da90c5164c678a7cf1e12bffa2e8 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4f61da90c5164c678a7cf1e12bffa2e8 2024-11-26T16:42:33,785 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4f61da90c5164c678a7cf1e12bffa2e8, entries=150, sequenceid=313, filesize=30.5 K 2024-11-26T16:42:33,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/21c9e12e45374566aef0ec6d5042227e as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/21c9e12e45374566aef0ec6d5042227e 2024-11-26T16:42:33,797 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/21c9e12e45374566aef0ec6d5042227e, entries=150, sequenceid=313, filesize=12.0 K 2024-11-26T16:42:33,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/7ac6d12a66394010a4998cb7009318b3 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/7ac6d12a66394010a4998cb7009318b3 2024-11-26T16:42:33,802 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/7ac6d12a66394010a4998cb7009318b3, entries=150, sequenceid=313, filesize=12.0 K 2024-11-26T16:42:33,803 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=93.93 KB/96180 for 28a7a40a9a40603189977c8da6b1d6a7 in 957ms, sequenceid=313, compaction requested=false 2024-11-26T16:42:33,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:33,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:33,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-26T16:42:33,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-26T16:42:33,807 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-26T16:42:33,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4900 sec 2024-11-26T16:42:33,809 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.4960 sec 2024-11-26T16:42:33,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:33,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28a7a40a9a40603189977c8da6b1d6a7 3/3 column families, dataSize=100.63 KB heapSize=285.38 KB 2024-11-26T16:42:33,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=A 2024-11-26T16:42:33,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:33,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=B 2024-11-26T16:42:33,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:33,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=C 2024-11-26T16:42:33,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:33,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f9a5d5e931d54784b8707d041e032ab5_28a7a40a9a40603189977c8da6b1d6a7 is 50, key is test_row_0/A:col10/1732639353787/Put/seqid=0 2024-11-26T16:42:33,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742232_1408 (size=14994) 2024-11-26T16:42:33,834 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,840 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f9a5d5e931d54784b8707d041e032ab5_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f9a5d5e931d54784b8707d041e032ab5_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:33,842 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/475cadb491144de2aa2dce4b153ea788, store: [table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:33,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/475cadb491144de2aa2dce4b153ea788 is 175, key is test_row_0/A:col10/1732639353787/Put/seqid=0 2024-11-26T16:42:33,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742233_1409 (size=39949) 2024-11-26T16:42:33,860 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/475cadb491144de2aa2dce4b153ea788 2024-11-26T16:42:33,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/3da3ee9403e44e34bb23a5095af5fae1 is 50, key is test_row_0/B:col10/1732639353787/Put/seqid=0 2024-11-26T16:42:33,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742234_1410 (size=12301) 2024-11-26T16:42:33,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/3da3ee9403e44e34bb23a5095af5fae1 2024-11-26T16:42:33,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/f6392bc0608247ecae02d7651024f646 is 50, key is test_row_0/C:col10/1732639353787/Put/seqid=0 2024-11-26T16:42:33,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742235_1411 (size=12301) 2024-11-26T16:42:33,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/f6392bc0608247ecae02d7651024f646 2024-11-26T16:42:33,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/475cadb491144de2aa2dce4b153ea788 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/475cadb491144de2aa2dce4b153ea788 2024-11-26T16:42:33,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/475cadb491144de2aa2dce4b153ea788, entries=200, sequenceid=348, filesize=39.0 K 2024-11-26T16:42:33,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/3da3ee9403e44e34bb23a5095af5fae1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/3da3ee9403e44e34bb23a5095af5fae1 2024-11-26T16:42:33,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/3da3ee9403e44e34bb23a5095af5fae1, entries=150, sequenceid=348, filesize=12.0 K 2024-11-26T16:42:33,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/f6392bc0608247ecae02d7651024f646 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/f6392bc0608247ecae02d7651024f646 2024-11-26T16:42:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/f6392bc0608247ecae02d7651024f646, entries=150, sequenceid=348, filesize=12.0 K 2024-11-26T16:42:33,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~285.33 KB/292176, currentSize=40.25 KB/41220 for 28a7a40a9a40603189977c8da6b1d6a7 in 119ms, sequenceid=348, compaction requested=true 2024-11-26T16:42:33,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:33,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:42:33,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:33,928 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:33,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:42:33,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:33,928 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:33,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:33,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:33,929 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:33,929 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/A is initiating minor compaction (all files) 2024-11-26T16:42:33,929 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/A in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:33,929 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:33,929 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/2d4b16b52bba41858a04ca5994a82224, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4f61da90c5164c678a7cf1e12bffa2e8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/475cadb491144de2aa2dce4b153ea788] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=100.6 K 2024-11-26T16:42:33,929 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/B is initiating minor compaction (all files) 2024-11-26T16:42:33,929 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:33,929 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. files: [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/2d4b16b52bba41858a04ca5994a82224, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4f61da90c5164c678a7cf1e12bffa2e8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/475cadb491144de2aa2dce4b153ea788] 2024-11-26T16:42:33,930 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/B in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:33,930 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b0f2cedd2e0c4aa59fe6834d14113c20, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/21c9e12e45374566aef0ec6d5042227e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/3da3ee9403e44e34bb23a5095af5fae1] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=36.6 K 2024-11-26T16:42:33,930 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting b0f2cedd2e0c4aa59fe6834d14113c20, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732639350045 2024-11-26T16:42:33,930 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d4b16b52bba41858a04ca5994a82224, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732639350045 2024-11-26T16:42:33,930 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 21c9e12e45374566aef0ec6d5042227e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732639351212 2024-11-26T16:42:33,930 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f61da90c5164c678a7cf1e12bffa2e8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732639351212 2024-11-26T16:42:33,931 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 3da3ee9403e44e34bb23a5095af5fae1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732639353716 2024-11-26T16:42:33,931 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 475cadb491144de2aa2dce4b153ea788, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732639353716 2024-11-26T16:42:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,948 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,950 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#B#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,950 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/2bf76341721c46ce987966826290fa5f is 50, key is test_row_0/B:col10/1732639353787/Put/seqid=0 2024-11-26T16:42:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,952 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126fcd6b786929a4da4bfcc5370e63b77a4_28a7a40a9a40603189977c8da6b1d6a7 store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,954 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126fcd6b786929a4da4bfcc5370e63b77a4_28a7a40a9a40603189977c8da6b1d6a7, store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,954 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126fcd6b786929a4da4bfcc5370e63b77a4_28a7a40a9a40603189977c8da6b1d6a7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742236_1412 (size=12983) 2024-11-26T16:42:33,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742237_1413 (size=4469) 2024-11-26T16:42:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:34,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28a7a40a9a40603189977c8da6b1d6a7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T16:42:34,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=A 2024-11-26T16:42:34,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:34,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=B 2024-11-26T16:42:34,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:34,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=C 2024-11-26T16:42:34,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112660429e82b6a3436a866e34cd444a64b1_28a7a40a9a40603189977c8da6b1d6a7 is 50, key is test_row_0/A:col10/1732639354004/Put/seqid=0 2024-11-26T16:42:34,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742238_1414 (size=12454) 2024-11-26T16:42:34,020 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,024 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112660429e82b6a3436a866e34cd444a64b1_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112660429e82b6a3436a866e34cd444a64b1_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,025 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/d63670f0f9464e5cbe5f3bf6c83789e1, store: [table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:34,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/d63670f0f9464e5cbe5f3bf6c83789e1 is 175, key is test_row_0/A:col10/1732639354004/Put/seqid=0 2024-11-26T16:42:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742239_1415 (size=31251) 2024-11-26T16:42:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:34,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639354157, max timestamp=1732639354201Num uniques 859; 2024-11-26T16:42:34,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:34,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639354157, max timestamp=1732639354201Num uniques 859; 2024-11-26T16:42:34,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639354157, max timestamp=1732639354201Num uniques 859; 2024-11-26T16:42:34,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:34,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:34,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:34,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639414332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:34,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:34,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639414333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:34,362 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/2bf76341721c46ce987966826290fa5f as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/2bf76341721c46ce987966826290fa5f 2024-11-26T16:42:34,367 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/B of 28a7a40a9a40603189977c8da6b1d6a7 into 2bf76341721c46ce987966826290fa5f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:34,367 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:34,367 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/B, priority=13, startTime=1732639353928; duration=0sec 2024-11-26T16:42:34,367 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:34,367 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:B 2024-11-26T16:42:34,367 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:34,368 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:34,368 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/C is initiating minor compaction (all files) 2024-11-26T16:42:34,368 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/C in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:34,368 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/6245aa7dd0ce4b84a701e0b600692727, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/7ac6d12a66394010a4998cb7009318b3, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/f6392bc0608247ecae02d7651024f646] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=36.6 K 2024-11-26T16:42:34,369 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 6245aa7dd0ce4b84a701e0b600692727, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732639350045 2024-11-26T16:42:34,369 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ac6d12a66394010a4998cb7009318b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732639351212 2024-11-26T16:42:34,369 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting f6392bc0608247ecae02d7651024f646, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732639353716 2024-11-26T16:42:34,377 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#C#compaction#348 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:34,377 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/d161d3ab13644605abee8118a7443e87 is 50, key is test_row_0/C:col10/1732639353787/Put/seqid=0 2024-11-26T16:42:34,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742240_1416 (size=12983) 2024-11-26T16:42:34,385 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#A#compaction#346 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:34,386 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/7450df8a5d8343f9bbc060bf0f054444 is 175, key is test_row_0/A:col10/1732639353787/Put/seqid=0 2024-11-26T16:42:34,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742241_1417 (size=31937) 2024-11-26T16:42:34,430 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=358, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/d63670f0f9464e5cbe5f3bf6c83789e1 2024-11-26T16:42:34,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/6bc0005e2ad14be7bb0f7e71fe26efe6 is 50, key is test_row_0/B:col10/1732639354004/Put/seqid=0 2024-11-26T16:42:34,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742242_1418 (size=9857) 2024-11-26T16:42:34,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:34,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639414451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:34,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639414451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:34,664 INFO [Thread-1530 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1530. 2024-11-26T16:42:34,664 DEBUG [Thread-1530 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:42:34,664 DEBUG [Thread-1530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:34,664 DEBUG [Thread-1530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:34,664 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:34,666 INFO [Thread-1524 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1524. 2024-11-26T16:42:34,666 DEBUG [Thread-1524 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:42:34,666 DEBUG [Thread-1524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:34,666 INFO [Thread-1526 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1526. 2024-11-26T16:42:34,666 DEBUG [Thread-1524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:34,667 DEBUG [Thread-1526 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:42:34,667 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:34,667 DEBUG [Thread-1526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:34,667 DEBUG [Thread-1526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:34,667 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:34,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639414671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:34,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639414671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:34,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:34,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:34,672 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:34,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:34,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:34,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:34,678 INFO [Thread-1522 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1522. 2024-11-26T16:42:34,678 DEBUG [Thread-1522 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:42:34,678 DEBUG [Thread-1522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:34,678 DEBUG [Thread-1522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:34,678 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:34,678 INFO [Thread-1528 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1528. 2024-11-26T16:42:34,678 DEBUG [Thread-1528 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:42:34,678 DEBUG [Thread-1528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:34,679 DEBUG [Thread-1528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:34,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:34,785 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/d161d3ab13644605abee8118a7443e87 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/d161d3ab13644605abee8118a7443e87 2024-11-26T16:42:34,789 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/C of 28a7a40a9a40603189977c8da6b1d6a7 into d161d3ab13644605abee8118a7443e87(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:34,789 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:34,789 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/C, priority=13, startTime=1732639353928; duration=0sec 2024-11-26T16:42:34,789 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:34,789 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:C 2024-11-26T16:42:34,793 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/7450df8a5d8343f9bbc060bf0f054444 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/7450df8a5d8343f9bbc060bf0f054444 2024-11-26T16:42:34,797 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/A of 28a7a40a9a40603189977c8da6b1d6a7 into 7450df8a5d8343f9bbc060bf0f054444(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:34,797 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:34,797 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/A, priority=13, startTime=1732639353928; duration=0sec 2024-11-26T16:42:34,797 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:34,797 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:A 2024-11-26T16:42:34,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/6bc0005e2ad14be7bb0f7e71fe26efe6 2024-11-26T16:42:34,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/e1e73b7570824f8fac166faaf4b3d355 is 50, key is test_row_0/C:col10/1732639354004/Put/seqid=0 2024-11-26T16:42:34,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742243_1419 (size=9857) 2024-11-26T16:42:34,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:34,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54632 deadline: 1732639414981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54582 deadline: 1732639414981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:34,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:34,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:34,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:34,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:34,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:34,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:42:35,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/e1e73b7570824f8fac166faaf4b3d355 2024-11-26T16:42:35,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/d63670f0f9464e5cbe5f3bf6c83789e1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d63670f0f9464e5cbe5f3bf6c83789e1 2024-11-26T16:42:35,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d63670f0f9464e5cbe5f3bf6c83789e1, entries=150, sequenceid=358, filesize=30.5 K 2024-11-26T16:42:35,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/6bc0005e2ad14be7bb0f7e71fe26efe6 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/6bc0005e2ad14be7bb0f7e71fe26efe6 2024-11-26T16:42:35,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/6bc0005e2ad14be7bb0f7e71fe26efe6, entries=100, sequenceid=358, filesize=9.6 K 2024-11-26T16:42:35,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/e1e73b7570824f8fac166faaf4b3d355 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/e1e73b7570824f8fac166faaf4b3d355 2024-11-26T16:42:35,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/e1e73b7570824f8fac166faaf4b3d355, entries=100, sequenceid=358, filesize=9.6 K 2024-11-26T16:42:35,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 28a7a40a9a40603189977c8da6b1d6a7 in 1260ms, sequenceid=358, compaction requested=false 2024-11-26T16:42:35,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:35,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-26T16:42:35,481 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:35,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28a7a40a9a40603189977c8da6b1d6a7 3/3 column families, dataSize=147.60 KB heapSize=425.53 KB 2024-11-26T16:42:35,493 INFO [Thread-1513 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1513. 2024-11-26T16:42:35,493 DEBUG [Thread-1513 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:42:35,493 DEBUG [Thread-1513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:35,493 INFO [Thread-1517 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1517. 2024-11-26T16:42:35,493 DEBUG [Thread-1517 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:42:35,493 DEBUG [Thread-1517 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:35,493 DEBUG [Thread-1513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:35,493 DEBUG [Thread-1517 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:35,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:35,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:35,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=A 2024-11-26T16:42:35,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:42:35,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=B 2024-11-26T16:42:35,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:42:35,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=C 2024-11-26T16:42:35,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:42:35,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261a9ee9a5780d49b1bdcdb464e6c99cea_28a7a40a9a40603189977c8da6b1d6a7 is 50, key is test_row_0/A:col10/1732639355491/Put/seqid=0 2024-11-26T16:42:35,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742244_1420 (size=12454) 2024-11-26T16:42:35,903 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:35,906 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261a9ee9a5780d49b1bdcdb464e6c99cea_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261a9ee9a5780d49b1bdcdb464e6c99cea_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:35,907 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/d3c95503942a48559b4424013c1cfe10, store: [table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:35,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/d3c95503942a48559b4424013c1cfe10 is 175, key is test_row_0/A:col10/1732639355491/Put/seqid=0 2024-11-26T16:42:35,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742245_1421 (size=31255) 2024-11-26T16:42:36,305 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T16:42:36,311 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=400, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/d3c95503942a48559b4424013c1cfe10 2024-11-26T16:42:36,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/b406149c925c416ab34e0c5a3c3dcc23 is 50, key is test_row_0/B:col10/1732639355491/Put/seqid=0 2024-11-26T16:42:36,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742246_1422 (size=12301) 2024-11-26T16:42:36,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/b406149c925c416ab34e0c5a3c3dcc23 2024-11-26T16:42:36,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/3cd1f96e2c7d454ab13c030011fe340a is 50, key is test_row_0/C:col10/1732639355491/Put/seqid=0 2024-11-26T16:42:36,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742247_1423 (size=12301) 2024-11-26T16:42:37,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/3cd1f96e2c7d454ab13c030011fe340a 2024-11-26T16:42:37,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/d3c95503942a48559b4424013c1cfe10 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d3c95503942a48559b4424013c1cfe10 2024-11-26T16:42:37,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d3c95503942a48559b4424013c1cfe10, entries=150, sequenceid=400, filesize=30.5 K 2024-11-26T16:42:37,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/b406149c925c416ab34e0c5a3c3dcc23 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b406149c925c416ab34e0c5a3c3dcc23 2024-11-26T16:42:37,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b406149c925c416ab34e0c5a3c3dcc23, entries=150, sequenceid=400, filesize=12.0 K 2024-11-26T16:42:37,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/3cd1f96e2c7d454ab13c030011fe340a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/3cd1f96e2c7d454ab13c030011fe340a 2024-11-26T16:42:37,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/3cd1f96e2c7d454ab13c030011fe340a, entries=150, sequenceid=400, filesize=12.0 K 2024-11-26T16:42:37,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~425.48 KB/435696, currentSize=0 B/0 for 28a7a40a9a40603189977c8da6b1d6a7 in 1654ms, sequenceid=400, compaction requested=true 2024-11-26T16:42:37,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:37,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:42:37,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:37,147 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:37,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:42:37,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:37,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28a7a40a9a40603189977c8da6b1d6a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:37,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:37,147 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:37,147 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:37,147 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/A is initiating minor compaction (all files) 2024-11-26T16:42:37,147 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:37,148 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/A in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:37,148 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/B is initiating minor compaction (all files) 2024-11-26T16:42:37,148 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/7450df8a5d8343f9bbc060bf0f054444, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d63670f0f9464e5cbe5f3bf6c83789e1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d3c95503942a48559b4424013c1cfe10] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=92.2 K 2024-11-26T16:42:37,148 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/B in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:37,148 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:37,148 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. files: [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/7450df8a5d8343f9bbc060bf0f054444, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d63670f0f9464e5cbe5f3bf6c83789e1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d3c95503942a48559b4424013c1cfe10] 2024-11-26T16:42:37,148 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/2bf76341721c46ce987966826290fa5f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/6bc0005e2ad14be7bb0f7e71fe26efe6, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b406149c925c416ab34e0c5a3c3dcc23] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=34.3 K 2024-11-26T16:42:37,148 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7450df8a5d8343f9bbc060bf0f054444, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732639353716 2024-11-26T16:42:37,148 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bf76341721c46ce987966826290fa5f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732639353716 2024-11-26T16:42:37,148 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bc0005e2ad14be7bb0f7e71fe26efe6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732639353855 2024-11-26T16:42:37,148 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting d63670f0f9464e5cbe5f3bf6c83789e1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732639353855 2024-11-26T16:42:37,149 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3c95503942a48559b4424013c1cfe10, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732639354301 2024-11-26T16:42:37,149 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting b406149c925c416ab34e0c5a3c3dcc23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732639354301 2024-11-26T16:42:37,159 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#B#compaction#354 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:37,160 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/5eb26b33730f42638f08c68df2b7d2c8 is 50, key is test_row_0/B:col10/1732639355491/Put/seqid=0 2024-11-26T16:42:37,163 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:37,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742248_1424 (size=13085) 2024-11-26T16:42:37,168 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126de86531688824585943a9c43ae9a0357_28a7a40a9a40603189977c8da6b1d6a7 store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:37,189 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126de86531688824585943a9c43ae9a0357_28a7a40a9a40603189977c8da6b1d6a7, store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:37,189 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126de86531688824585943a9c43ae9a0357_28a7a40a9a40603189977c8da6b1d6a7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:37,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742249_1425 (size=4469) 2024-11-26T16:42:37,570 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/5eb26b33730f42638f08c68df2b7d2c8 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/5eb26b33730f42638f08c68df2b7d2c8 2024-11-26T16:42:37,574 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/B of 28a7a40a9a40603189977c8da6b1d6a7 into 5eb26b33730f42638f08c68df2b7d2c8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:37,574 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:37,574 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/B, priority=13, startTime=1732639357147; duration=0sec 2024-11-26T16:42:37,574 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:37,574 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:B 2024-11-26T16:42:37,574 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:37,575 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:37,575 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 28a7a40a9a40603189977c8da6b1d6a7/C is initiating minor compaction (all files) 2024-11-26T16:42:37,576 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28a7a40a9a40603189977c8da6b1d6a7/C in TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:37,576 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/d161d3ab13644605abee8118a7443e87, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/e1e73b7570824f8fac166faaf4b3d355, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/3cd1f96e2c7d454ab13c030011fe340a] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp, totalSize=34.3 K 2024-11-26T16:42:37,576 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting d161d3ab13644605abee8118a7443e87, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732639353716 2024-11-26T16:42:37,576 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting e1e73b7570824f8fac166faaf4b3d355, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732639353855 2024-11-26T16:42:37,576 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cd1f96e2c7d454ab13c030011fe340a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732639354301 2024-11-26T16:42:37,584 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#C#compaction#356 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:37,584 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/ea6e072e19184df2a43eb70486d1af7d is 50, key is test_row_0/C:col10/1732639355491/Put/seqid=0 2024-11-26T16:42:37,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742250_1426 (size=13085) 2024-11-26T16:42:37,594 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28a7a40a9a40603189977c8da6b1d6a7#A#compaction#355 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:37,595 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/62246b648f7848429d4bb3219284842f is 175, key is test_row_0/A:col10/1732639355491/Put/seqid=0 2024-11-26T16:42:37,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742251_1427 (size=32039) 2024-11-26T16:42:37,602 INFO [Thread-1511 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1511. 2024-11-26T16:42:37,602 DEBUG [Thread-1511 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:42:37,602 DEBUG [Thread-1511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:37,603 DEBUG [Thread-1511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:37,603 INFO [Thread-1519 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1519. 2024-11-26T16:42:37,603 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:37,603 DEBUG [Thread-1519 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:42:37,603 DEBUG [Thread-1519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:37,603 DEBUG [Thread-1519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:37,603 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:37,643 INFO [Thread-1515 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1515. 2024-11-26T16:42:37,643 DEBUG [Thread-1515 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:42:37,643 DEBUG [Thread-1515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:37,643 DEBUG [Thread-1515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:37,643 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:37,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T16:42:37,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-26T16:42:37,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 178 2024-11-26T16:42:37,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-26T16:42:37,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-26T16:42:37,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-26T16:42:37,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T16:42:37,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T16:42:37,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 886 2024-11-26T16:42:37,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2653 rows 2024-11-26T16:42:37,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 888 2024-11-26T16:42:37,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2662 rows 2024-11-26T16:42:37,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 887 2024-11-26T16:42:37,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2658 rows 2024-11-26T16:42:37,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 882 2024-11-26T16:42:37,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2645 rows 2024-11-26T16:42:37,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 882 2024-11-26T16:42:37,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2643 rows 2024-11-26T16:42:37,644 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T16:42:37,644 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T16:42:37,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:37,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:37,644 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:42:37,645 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T16:42:37,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T16:42:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T16:42:37,647 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639357647"}]},"ts":"1732639357647"} 2024-11-26T16:42:37,648 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T16:42:37,649 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T16:42:37,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T16:42:37,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=28a7a40a9a40603189977c8da6b1d6a7, UNASSIGN}] 2024-11-26T16:42:37,651 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=28a7a40a9a40603189977c8da6b1d6a7, UNASSIGN 2024-11-26T16:42:37,652 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=28a7a40a9a40603189977c8da6b1d6a7, regionState=CLOSING, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:37,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=28a7a40a9a40603189977c8da6b1d6a7, UNASSIGN because future has completed 2024-11-26T16:42:37,653 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T16:42:37,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure 28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134}] 2024-11-26T16:42:37,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T16:42:37,805 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(122): Close 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:37,805 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T16:42:37,805 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1722): Closing 28a7a40a9a40603189977c8da6b1d6a7, disabling compactions & flushes 2024-11-26T16:42:37,805 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T16:42:37,992 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/ea6e072e19184df2a43eb70486d1af7d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/ea6e072e19184df2a43eb70486d1af7d 2024-11-26T16:42:37,996 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/C of 28a7a40a9a40603189977c8da6b1d6a7 into ea6e072e19184df2a43eb70486d1af7d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:37,996 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:37,996 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/C, priority=13, startTime=1732639357147; duration=0sec 2024-11-26T16:42:37,996 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:37,996 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:C 2024-11-26T16:42:38,001 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/62246b648f7848429d4bb3219284842f as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/62246b648f7848429d4bb3219284842f 2024-11-26T16:42:38,004 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28a7a40a9a40603189977c8da6b1d6a7/A of 28a7a40a9a40603189977c8da6b1d6a7 into 62246b648f7848429d4bb3219284842f(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:38,004 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28a7a40a9a40603189977c8da6b1d6a7: 2024-11-26T16:42:38,004 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:38,004 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., storeName=28a7a40a9a40603189977c8da6b1d6a7/A, priority=13, startTime=1732639357146; duration=0sec 2024-11-26T16:42:38,004 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:38,004 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:38,005 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28a7a40a9a40603189977c8da6b1d6a7:A 2024-11-26T16:42:38,005 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. after waiting 0 ms 2024-11-26T16:42:38,005 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:38,005 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(2902): Flushing 28a7a40a9a40603189977c8da6b1d6a7 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-26T16:42:38,005 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=A 2024-11-26T16:42:38,005 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:38,005 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=B 2024-11-26T16:42:38,005 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:38,005 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 28a7a40a9a40603189977c8da6b1d6a7, store=C 2024-11-26T16:42:38,005 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:38,010 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265bca3e18d2b64e92a3fb65706d4856ee_28a7a40a9a40603189977c8da6b1d6a7 is 50, key is test_row_1/A:col10/1732639357601/Put/seqid=0 2024-11-26T16:42:38,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742252_1428 (size=9914) 2024-11-26T16:42:38,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T16:42:38,413 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:38,417 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411265bca3e18d2b64e92a3fb65706d4856ee_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265bca3e18d2b64e92a3fb65706d4856ee_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:38,417 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/abe7bd80e8f64203a512b6590fafbc7e, store: [table=TestAcidGuarantees family=A region=28a7a40a9a40603189977c8da6b1d6a7] 2024-11-26T16:42:38,418 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/abe7bd80e8f64203a512b6590fafbc7e is 175, key is test_row_1/A:col10/1732639357601/Put/seqid=0 2024-11-26T16:42:38,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742253_1429 (size=22561) 2024-11-26T16:42:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T16:42:38,822 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=409, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/abe7bd80e8f64203a512b6590fafbc7e 2024-11-26T16:42:38,829 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/a3b3651706ac47ab890f82c1f2ea96a3 is 50, key is test_row_1/B:col10/1732639357601/Put/seqid=0 2024-11-26T16:42:38,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742254_1430 (size=9857) 2024-11-26T16:42:39,233 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/a3b3651706ac47ab890f82c1f2ea96a3 2024-11-26T16:42:39,239 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/1a45ed9bddcb4deb943547a160c6c723 is 50, key is test_row_1/C:col10/1732639357601/Put/seqid=0 2024-11-26T16:42:39,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742255_1431 (size=9857) 2024-11-26T16:42:39,642 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/1a45ed9bddcb4deb943547a160c6c723 2024-11-26T16:42:39,646 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/A/abe7bd80e8f64203a512b6590fafbc7e as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/abe7bd80e8f64203a512b6590fafbc7e 2024-11-26T16:42:39,650 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/abe7bd80e8f64203a512b6590fafbc7e, entries=100, sequenceid=409, filesize=22.0 K 2024-11-26T16:42:39,650 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/B/a3b3651706ac47ab890f82c1f2ea96a3 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/a3b3651706ac47ab890f82c1f2ea96a3 2024-11-26T16:42:39,653 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/a3b3651706ac47ab890f82c1f2ea96a3, entries=100, sequenceid=409, filesize=9.6 K 2024-11-26T16:42:39,654 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/.tmp/C/1a45ed9bddcb4deb943547a160c6c723 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/1a45ed9bddcb4deb943547a160c6c723 2024-11-26T16:42:39,657 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/1a45ed9bddcb4deb943547a160c6c723, entries=100, sequenceid=409, filesize=9.6 K 2024-11-26T16:42:39,657 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 28a7a40a9a40603189977c8da6b1d6a7 in 1652ms, sequenceid=409, compaction requested=false 2024-11-26T16:42:39,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/5be0f4eb416547f2a437b8339cd0d534, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4a0d00113f99499ca67027eb1f013f11, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/f728f64eaaeb4e96acfb0aac28da6beb, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/c740d8e0e95a4e3d8c1b7e2f80ed3cdd, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/078f850a7090472b8efcaab09207c111, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/712758c03998454186aae6adb9fe84ae, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/5196fdbd647f46b398e1f4352a142fc8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/b77e1ae5e42542a6a61b75ba02d333e4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4d48e57613ca43f9bff7bb7912f5ea65, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/77ca0b544a9f44b5b89fb749b4284999, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/68a655ada1b743368096a99afea47e52, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/da1776a7f24d47f79ca5c85940ea80cd, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d2dbdbe08d8d4d3a931e559adab119db, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/3499529ef1964ec3aaf948695689915f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/de5b8fab077d466d88ccfbbcc53a19cf, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/ad20325edc22408182b6b244339903b1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/8d092d07f1db4756978d659ac61b1ee9, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/2d4b16b52bba41858a04ca5994a82224, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4f61da90c5164c678a7cf1e12bffa2e8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/475cadb491144de2aa2dce4b153ea788, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/7450df8a5d8343f9bbc060bf0f054444, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d63670f0f9464e5cbe5f3bf6c83789e1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d3c95503942a48559b4424013c1cfe10] to archive 2024-11-26T16:42:39,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T16:42:39,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/5be0f4eb416547f2a437b8339cd0d534 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/5be0f4eb416547f2a437b8339cd0d534 2024-11-26T16:42:39,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4a0d00113f99499ca67027eb1f013f11 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4a0d00113f99499ca67027eb1f013f11 2024-11-26T16:42:39,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/f728f64eaaeb4e96acfb0aac28da6beb to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/f728f64eaaeb4e96acfb0aac28da6beb 2024-11-26T16:42:39,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/c740d8e0e95a4e3d8c1b7e2f80ed3cdd to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/c740d8e0e95a4e3d8c1b7e2f80ed3cdd 2024-11-26T16:42:39,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/078f850a7090472b8efcaab09207c111 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/078f850a7090472b8efcaab09207c111 2024-11-26T16:42:39,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/712758c03998454186aae6adb9fe84ae to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/712758c03998454186aae6adb9fe84ae 2024-11-26T16:42:39,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/5196fdbd647f46b398e1f4352a142fc8 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/5196fdbd647f46b398e1f4352a142fc8 2024-11-26T16:42:39,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/b77e1ae5e42542a6a61b75ba02d333e4 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/b77e1ae5e42542a6a61b75ba02d333e4 2024-11-26T16:42:39,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4d48e57613ca43f9bff7bb7912f5ea65 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4d48e57613ca43f9bff7bb7912f5ea65 2024-11-26T16:42:39,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/77ca0b544a9f44b5b89fb749b4284999 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/77ca0b544a9f44b5b89fb749b4284999 2024-11-26T16:42:39,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/68a655ada1b743368096a99afea47e52 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/68a655ada1b743368096a99afea47e52 2024-11-26T16:42:39,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/da1776a7f24d47f79ca5c85940ea80cd to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/da1776a7f24d47f79ca5c85940ea80cd 2024-11-26T16:42:39,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d2dbdbe08d8d4d3a931e559adab119db to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d2dbdbe08d8d4d3a931e559adab119db 2024-11-26T16:42:39,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/3499529ef1964ec3aaf948695689915f to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/3499529ef1964ec3aaf948695689915f 2024-11-26T16:42:39,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/de5b8fab077d466d88ccfbbcc53a19cf to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/de5b8fab077d466d88ccfbbcc53a19cf 2024-11-26T16:42:39,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/ad20325edc22408182b6b244339903b1 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/ad20325edc22408182b6b244339903b1 2024-11-26T16:42:39,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/8d092d07f1db4756978d659ac61b1ee9 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/8d092d07f1db4756978d659ac61b1ee9 2024-11-26T16:42:39,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/2d4b16b52bba41858a04ca5994a82224 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/2d4b16b52bba41858a04ca5994a82224 2024-11-26T16:42:39,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4f61da90c5164c678a7cf1e12bffa2e8 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/4f61da90c5164c678a7cf1e12bffa2e8 2024-11-26T16:42:39,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/475cadb491144de2aa2dce4b153ea788 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/475cadb491144de2aa2dce4b153ea788 2024-11-26T16:42:39,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/7450df8a5d8343f9bbc060bf0f054444 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/7450df8a5d8343f9bbc060bf0f054444 2024-11-26T16:42:39,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d63670f0f9464e5cbe5f3bf6c83789e1 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d63670f0f9464e5cbe5f3bf6c83789e1 2024-11-26T16:42:39,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d3c95503942a48559b4424013c1cfe10 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/d3c95503942a48559b4424013c1cfe10 2024-11-26T16:42:39,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/560ccab703b14672ac4ffccc7cea038b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/41f26ffe39374165a7dc76b4aa3af301, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/96250f39c6c842a8a26201019e69c54d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/38c5e885e42c4ee785b5d4594216a433, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/c74837f986c044959f9f8f252be9e7cf, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/a8c7ea83aee44797b9d7edb7e013882e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/27874bbe1f7c46af955dacc6787d467a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/a04ba914cb0f41508f60fee1d8a27afc, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/ebf3fe17a4284e4193fbd8cb8881cf14, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/f308a78d697a49f79a98efad5c123b6a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b4d64309ddc347ec96b5642e1b4d0bd3, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/2e12a550bd1f47568537b58266e29fd4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b545789149364af88d107b5bf28c0f2c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/718a4102635641ba82511174ce1d7770, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/83a9fc5d72d7417885e9f5c5a32741f7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/0acdf0149ef4446bb5156fb893efcf70, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b0f2cedd2e0c4aa59fe6834d14113c20, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/592d1fd305b7406ab1eb6cce60d20f4a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/21c9e12e45374566aef0ec6d5042227e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/2bf76341721c46ce987966826290fa5f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/3da3ee9403e44e34bb23a5095af5fae1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/6bc0005e2ad14be7bb0f7e71fe26efe6, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b406149c925c416ab34e0c5a3c3dcc23] to archive 2024-11-26T16:42:39,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T16:42:39,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/560ccab703b14672ac4ffccc7cea038b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/560ccab703b14672ac4ffccc7cea038b 2024-11-26T16:42:39,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/41f26ffe39374165a7dc76b4aa3af301 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/41f26ffe39374165a7dc76b4aa3af301 2024-11-26T16:42:39,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/96250f39c6c842a8a26201019e69c54d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/96250f39c6c842a8a26201019e69c54d 2024-11-26T16:42:39,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/38c5e885e42c4ee785b5d4594216a433 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/38c5e885e42c4ee785b5d4594216a433 2024-11-26T16:42:39,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/c74837f986c044959f9f8f252be9e7cf to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/c74837f986c044959f9f8f252be9e7cf 2024-11-26T16:42:39,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/a8c7ea83aee44797b9d7edb7e013882e to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/a8c7ea83aee44797b9d7edb7e013882e 2024-11-26T16:42:39,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/27874bbe1f7c46af955dacc6787d467a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/27874bbe1f7c46af955dacc6787d467a 2024-11-26T16:42:39,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/a04ba914cb0f41508f60fee1d8a27afc to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/a04ba914cb0f41508f60fee1d8a27afc 2024-11-26T16:42:39,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/ebf3fe17a4284e4193fbd8cb8881cf14 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/ebf3fe17a4284e4193fbd8cb8881cf14 2024-11-26T16:42:39,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/f308a78d697a49f79a98efad5c123b6a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/f308a78d697a49f79a98efad5c123b6a 2024-11-26T16:42:39,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b4d64309ddc347ec96b5642e1b4d0bd3 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b4d64309ddc347ec96b5642e1b4d0bd3 2024-11-26T16:42:39,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/2e12a550bd1f47568537b58266e29fd4 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/2e12a550bd1f47568537b58266e29fd4 2024-11-26T16:42:39,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b545789149364af88d107b5bf28c0f2c to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b545789149364af88d107b5bf28c0f2c 2024-11-26T16:42:39,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/718a4102635641ba82511174ce1d7770 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/718a4102635641ba82511174ce1d7770 2024-11-26T16:42:39,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/83a9fc5d72d7417885e9f5c5a32741f7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/83a9fc5d72d7417885e9f5c5a32741f7 2024-11-26T16:42:39,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/0acdf0149ef4446bb5156fb893efcf70 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/0acdf0149ef4446bb5156fb893efcf70 2024-11-26T16:42:39,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b0f2cedd2e0c4aa59fe6834d14113c20 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b0f2cedd2e0c4aa59fe6834d14113c20 2024-11-26T16:42:39,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/592d1fd305b7406ab1eb6cce60d20f4a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/592d1fd305b7406ab1eb6cce60d20f4a 2024-11-26T16:42:39,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/21c9e12e45374566aef0ec6d5042227e to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/21c9e12e45374566aef0ec6d5042227e 2024-11-26T16:42:39,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/2bf76341721c46ce987966826290fa5f to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/2bf76341721c46ce987966826290fa5f 2024-11-26T16:42:39,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/3da3ee9403e44e34bb23a5095af5fae1 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/3da3ee9403e44e34bb23a5095af5fae1 2024-11-26T16:42:39,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/6bc0005e2ad14be7bb0f7e71fe26efe6 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/6bc0005e2ad14be7bb0f7e71fe26efe6 2024-11-26T16:42:39,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b406149c925c416ab34e0c5a3c3dcc23 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/b406149c925c416ab34e0c5a3c3dcc23 2024-11-26T16:42:39,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/9b818ef5b36e468d8c96e028602f2ed4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/a48c36efa2454fe295386b2edfac4895, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/1bfc37afe4ca4fb2b0056204cc5b2d0d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/7d37c5055ffb4c77b0223432756a8d4b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/c85a09885667457cb3a2d83d171da24d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/c2e847be82134be3b5f9a825a66ca895, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/a7789461c8324d828b4d77f4abe2477b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/05380e1ff3db43aea4de8d4a88f7e734, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/050888ee3d3a46bbb22709d2de63ef06, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/6f8c41c23fef4985b8c18a49e465eb9d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/00d0c4e3e21b4ee88dcaa1f528cb2b11, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/e1124c7ffabe4b0e930d50cb34c4ec63, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/3c10a0311d28449e8bc549e3e1d35537, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/11ac9cfb534b4d75a78bee76d84b0989, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/576710fa56f940c59175c8393c452b69, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/957bf89f20dd4cdeb75b2a71e2e26da2, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/6245aa7dd0ce4b84a701e0b600692727, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/1a97f16698834b6db195e9eb9e0691e7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/7ac6d12a66394010a4998cb7009318b3, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/d161d3ab13644605abee8118a7443e87, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/f6392bc0608247ecae02d7651024f646, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/e1e73b7570824f8fac166faaf4b3d355, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/3cd1f96e2c7d454ab13c030011fe340a] to archive 2024-11-26T16:42:39,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T16:42:39,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/9b818ef5b36e468d8c96e028602f2ed4 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/9b818ef5b36e468d8c96e028602f2ed4 2024-11-26T16:42:39,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/a48c36efa2454fe295386b2edfac4895 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/a48c36efa2454fe295386b2edfac4895 2024-11-26T16:42:39,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/1bfc37afe4ca4fb2b0056204cc5b2d0d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/1bfc37afe4ca4fb2b0056204cc5b2d0d 2024-11-26T16:42:39,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/7d37c5055ffb4c77b0223432756a8d4b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/7d37c5055ffb4c77b0223432756a8d4b 2024-11-26T16:42:39,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/c85a09885667457cb3a2d83d171da24d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/c85a09885667457cb3a2d83d171da24d 2024-11-26T16:42:39,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/c2e847be82134be3b5f9a825a66ca895 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/c2e847be82134be3b5f9a825a66ca895 2024-11-26T16:42:39,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/a7789461c8324d828b4d77f4abe2477b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/a7789461c8324d828b4d77f4abe2477b 2024-11-26T16:42:39,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/05380e1ff3db43aea4de8d4a88f7e734 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/05380e1ff3db43aea4de8d4a88f7e734 2024-11-26T16:42:39,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/050888ee3d3a46bbb22709d2de63ef06 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/050888ee3d3a46bbb22709d2de63ef06 2024-11-26T16:42:39,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/6f8c41c23fef4985b8c18a49e465eb9d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/6f8c41c23fef4985b8c18a49e465eb9d 2024-11-26T16:42:39,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/00d0c4e3e21b4ee88dcaa1f528cb2b11 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/00d0c4e3e21b4ee88dcaa1f528cb2b11 2024-11-26T16:42:39,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/e1124c7ffabe4b0e930d50cb34c4ec63 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/e1124c7ffabe4b0e930d50cb34c4ec63 2024-11-26T16:42:39,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/3c10a0311d28449e8bc549e3e1d35537 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/3c10a0311d28449e8bc549e3e1d35537 2024-11-26T16:42:39,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/11ac9cfb534b4d75a78bee76d84b0989 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/11ac9cfb534b4d75a78bee76d84b0989 2024-11-26T16:42:39,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/576710fa56f940c59175c8393c452b69 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/576710fa56f940c59175c8393c452b69 2024-11-26T16:42:39,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/957bf89f20dd4cdeb75b2a71e2e26da2 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/957bf89f20dd4cdeb75b2a71e2e26da2 2024-11-26T16:42:39,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/6245aa7dd0ce4b84a701e0b600692727 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/6245aa7dd0ce4b84a701e0b600692727 2024-11-26T16:42:39,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/1a97f16698834b6db195e9eb9e0691e7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/1a97f16698834b6db195e9eb9e0691e7 2024-11-26T16:42:39,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/7ac6d12a66394010a4998cb7009318b3 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/7ac6d12a66394010a4998cb7009318b3 2024-11-26T16:42:39,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/d161d3ab13644605abee8118a7443e87 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/d161d3ab13644605abee8118a7443e87 2024-11-26T16:42:39,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/f6392bc0608247ecae02d7651024f646 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/f6392bc0608247ecae02d7651024f646 2024-11-26T16:42:39,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/e1e73b7570824f8fac166faaf4b3d355 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/e1e73b7570824f8fac166faaf4b3d355 2024-11-26T16:42:39,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/3cd1f96e2c7d454ab13c030011fe340a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/3cd1f96e2c7d454ab13c030011fe340a 2024-11-26T16:42:39,727 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/recovered.edits/412.seqid, newMaxSeqId=412, maxSeqId=4 2024-11-26T16:42:39,728 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. 2024-11-26T16:42:39,728 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1676): Region close journal for 28a7a40a9a40603189977c8da6b1d6a7: Waiting for close lock at 1732639357805Running coprocessor pre-close hooks at 1732639357805Disabling compacts and flushes for region at 1732639357805Disabling writes for close at 1732639358005 (+200 ms)Obtaining lock to block concurrent updates at 1732639358005Preparing flush snapshotting stores in 28a7a40a9a40603189977c8da6b1d6a7 at 1732639358005Finished memstore snapshotting TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732639358005Flushing stores of TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7. at 1732639358005Flushing 28a7a40a9a40603189977c8da6b1d6a7/A: creating writer at 1732639358005Flushing mob file 28a7a40a9a40603189977c8da6b1d6a7/A: appending metadata at 1732639358009 (+4 ms)Flushing mob file 28a7a40a9a40603189977c8da6b1d6a7/A: closing flushed file at 1732639358009Flushing 28a7a40a9a40603189977c8da6b1d6a7/A: appending metadata at 1732639358417 (+408 ms)Flushing 28a7a40a9a40603189977c8da6b1d6a7/A: closing flushed file at 1732639358417Flushing 28a7a40a9a40603189977c8da6b1d6a7/B: creating writer at 1732639358826 (+409 ms)Flushing 28a7a40a9a40603189977c8da6b1d6a7/B: appending metadata at 1732639358828 (+2 ms)Flushing 28a7a40a9a40603189977c8da6b1d6a7/B: closing flushed file at 1732639358828Flushing 28a7a40a9a40603189977c8da6b1d6a7/C: creating writer at 1732639359236 (+408 ms)Flushing 28a7a40a9a40603189977c8da6b1d6a7/C: appending metadata at 1732639359238 (+2 ms)Flushing 28a7a40a9a40603189977c8da6b1d6a7/C: closing flushed file at 1732639359238Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44ef1b27: reopening flushed file at 1732639359646 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21f233ec: reopening flushed file at 1732639359650 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cfddeba: reopening flushed file at 1732639359653 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 28a7a40a9a40603189977c8da6b1d6a7 in 1652ms, sequenceid=409, compaction requested=false at 1732639359657 (+4 ms)Writing region close event to WAL at 1732639359724 (+67 ms)Running coprocessor post-close hooks at 1732639359728 (+4 ms)Closed at 1732639359728 2024-11-26T16:42:39,730 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(157): Closed 28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,730 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=28a7a40a9a40603189977c8da6b1d6a7, regionState=CLOSED 2024-11-26T16:42:39,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure 28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 because future has completed 2024-11-26T16:42:39,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-26T16:42:39,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseRegionProcedure 28a7a40a9a40603189977c8da6b1d6a7, server=2dedfe7466e8,38109,1732639238134 in 2.0790 sec 2024-11-26T16:42:39,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-26T16:42:39,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=28a7a40a9a40603189977c8da6b1d6a7, UNASSIGN in 2.0840 sec 2024-11-26T16:42:39,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-26T16:42:39,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0870 sec 2024-11-26T16:42:39,739 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639359739"}]},"ts":"1732639359739"} 2024-11-26T16:42:39,740 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T16:42:39,740 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T16:42:39,742 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0960 sec 2024-11-26T16:42:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-26T16:42:39,791 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:39,792 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T16:42:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:42:39,794 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:42:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T16:42:39,794 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=107, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:42:39,796 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,798 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A, FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B, FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C, FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/recovered.edits] 2024-11-26T16:42:39,800 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/62246b648f7848429d4bb3219284842f to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/62246b648f7848429d4bb3219284842f 2024-11-26T16:42:39,801 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/abe7bd80e8f64203a512b6590fafbc7e to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/A/abe7bd80e8f64203a512b6590fafbc7e 2024-11-26T16:42:39,803 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/5eb26b33730f42638f08c68df2b7d2c8 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/5eb26b33730f42638f08c68df2b7d2c8 2024-11-26T16:42:39,804 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/a3b3651706ac47ab890f82c1f2ea96a3 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/B/a3b3651706ac47ab890f82c1f2ea96a3 2024-11-26T16:42:39,806 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/1a45ed9bddcb4deb943547a160c6c723 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/1a45ed9bddcb4deb943547a160c6c723 2024-11-26T16:42:39,807 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/ea6e072e19184df2a43eb70486d1af7d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/C/ea6e072e19184df2a43eb70486d1af7d 2024-11-26T16:42:39,809 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/recovered.edits/412.seqid to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7/recovered.edits/412.seqid 2024-11-26T16:42:39,810 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,810 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T16:42:39,811 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T16:42:39,811 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T16:42:39,814 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260389a7be605440089f3a90598c4de0a0_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411260389a7be605440089f3a90598c4de0a0_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,815 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261a9ee9a5780d49b1bdcdb464e6c99cea_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261a9ee9a5780d49b1bdcdb464e6c99cea_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,816 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262750b5da718b4addaee97866b248510e_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411262750b5da718b4addaee97866b248510e_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,817 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112627f359b915414e87a4635fc37ed73e81_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112627f359b915414e87a4635fc37ed73e81_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,818 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112641ec1e8c16534aaa8b51428ac4189e69_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112641ec1e8c16534aaa8b51428ac4189e69_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,819 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112647e471d30b884e87a2102b10681cd1e1_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112647e471d30b884e87a2102b10681cd1e1_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,820 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264f1dbc521ef5412fb91cddb1473e9e86_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411264f1dbc521ef5412fb91cddb1473e9e86_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,821 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112654c68a8850344f20b241deffd6daecbb_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112654c68a8850344f20b241deffd6daecbb_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,821 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265bca3e18d2b64e92a3fb65706d4856ee_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411265bca3e18d2b64e92a3fb65706d4856ee_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,822 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112660429e82b6a3436a866e34cd444a64b1_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112660429e82b6a3436a866e34cd444a64b1_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,823 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266c9ecb68d8d64e0ea788de7f19e7cd28_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411266c9ecb68d8d64e0ea788de7f19e7cd28_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,824 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112694fc7b7e1cda485ba1aff797cf470b2d_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112694fc7b7e1cda485ba1aff797cf470b2d_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,825 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126967976dd31f5458eae894bea313f13b0_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126967976dd31f5458eae894bea313f13b0_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,826 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a456df9a2c6d48dbb56a568269050b74_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a456df9a2c6d48dbb56a568269050b74_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,827 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a5f9daea894347778cc696e114e63912_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126a5f9daea894347778cc696e114e63912_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,828 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bb65fc19e1f349968125a26786e489cf_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126bb65fc19e1f349968125a26786e489cf_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,829 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ead47d37c3624b47a01bf2b615249717_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126ead47d37c3624b47a01bf2b615249717_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,830 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f9a5d5e931d54784b8707d041e032ab5_28a7a40a9a40603189977c8da6b1d6a7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126f9a5d5e931d54784b8707d041e032ab5_28a7a40a9a40603189977c8da6b1d6a7 2024-11-26T16:42:39,831 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T16:42:39,833 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=107, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:42:39,835 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T16:42:39,837 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T16:42:39,838 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=107, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:42:39,838 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T16:42:39,838 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732639359838"}]},"ts":"9223372036854775807"} 2024-11-26T16:42:39,839 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T16:42:39,839 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 28a7a40a9a40603189977c8da6b1d6a7, NAME => 'TestAcidGuarantees,,1732639332545.28a7a40a9a40603189977c8da6b1d6a7.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T16:42:39,840 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T16:42:39,840 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732639359840"}]},"ts":"9223372036854775807"} 2024-11-26T16:42:39,841 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T16:42:39,842 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=107, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:42:39,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 50 msec 2024-11-26T16:42:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-26T16:42:39,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T16:42:39,902 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:39,912 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=253 (was 250) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2059946220_22 at /127.0.0.1:60984 [Waiting for operation #98] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2059946220_22 at /127.0.0.1:60986 [Waiting for operation #81] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1947582375_22 at /127.0.0.1:55688 [Waiting for operation #634] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1947582375_22 at /127.0.0.1:32916 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2059946220_22 at /127.0.0.1:32876 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1947582375_22 at /127.0.0.1:54996 [Waiting for operation #499] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=465 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=421 (was 403) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5191 (was 5311) 2024-11-26T16:42:39,921 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=253, OpenFileDescriptor=465, MaxFileDescriptor=1048576, SystemLoadAverage=421, ProcessCount=11, AvailableMemoryMB=5187 2024-11-26T16:42:39,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T16:42:39,923 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T16:42:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T16:42:39,925 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T16:42:39,925 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:39,925 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 108 2024-11-26T16:42:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-26T16:42:39,926 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T16:42:39,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742256_1432 (size=1061) 2024-11-26T16:42:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-26T16:42:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-26T16:42:40,247 WARN [regionserver/2dedfe7466e8:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 3, running: 1 2024-11-26T16:42:40,333 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711 2024-11-26T16:42:40,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742257_1433 (size=53) 2024-11-26T16:42:40,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-26T16:42:40,738 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T16:42:40,738 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 42f6c759eca28a5c6a511cb6fa26803c, disabling compactions & flushes 2024-11-26T16:42:40,738 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:40,738 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:40,738 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. after waiting 0 ms 2024-11-26T16:42:40,738 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:40,738 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:40,738 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 42f6c759eca28a5c6a511cb6fa26803c: Waiting for close lock at 1732639360738Disabling compacts and flushes for region at 1732639360738Disabling writes for close at 1732639360738Writing region close event to WAL at 1732639360738Closed at 1732639360738 2024-11-26T16:42:40,739 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T16:42:40,740 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732639360739"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732639360739"}]},"ts":"1732639360739"} 2024-11-26T16:42:40,741 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T16:42:40,741 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T16:42:40,742 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639360741"}]},"ts":"1732639360741"} 2024-11-26T16:42:40,743 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T16:42:40,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42f6c759eca28a5c6a511cb6fa26803c, ASSIGN}] 2024-11-26T16:42:40,744 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42f6c759eca28a5c6a511cb6fa26803c, ASSIGN 2024-11-26T16:42:40,744 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42f6c759eca28a5c6a511cb6fa26803c, ASSIGN; state=OFFLINE, location=2dedfe7466e8,38109,1732639238134; forceNewPlan=false, retain=false 2024-11-26T16:42:40,895 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=42f6c759eca28a5c6a511cb6fa26803c, regionState=OPENING, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:40,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42f6c759eca28a5c6a511cb6fa26803c, ASSIGN because future has completed 2024-11-26T16:42:40,897 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134}] 2024-11-26T16:42:41,051 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:41,051 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7752): Opening region: {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} 2024-11-26T16:42:41,052 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:41,052 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T16:42:41,052 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7794): checking encryption for 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:41,052 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7797): checking classloading for 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:41,053 INFO [StoreOpener-42f6c759eca28a5c6a511cb6fa26803c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:41,054 INFO [StoreOpener-42f6c759eca28a5c6a511cb6fa26803c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:42:41,055 INFO [StoreOpener-42f6c759eca28a5c6a511cb6fa26803c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42f6c759eca28a5c6a511cb6fa26803c columnFamilyName A 2024-11-26T16:42:41,055 DEBUG [StoreOpener-42f6c759eca28a5c6a511cb6fa26803c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:41,055 INFO [StoreOpener-42f6c759eca28a5c6a511cb6fa26803c-1 {}] regionserver.HStore(327): Store=42f6c759eca28a5c6a511cb6fa26803c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:42:41,055 INFO [StoreOpener-42f6c759eca28a5c6a511cb6fa26803c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:41,056 INFO [StoreOpener-42f6c759eca28a5c6a511cb6fa26803c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:42:41,056 INFO [StoreOpener-42f6c759eca28a5c6a511cb6fa26803c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42f6c759eca28a5c6a511cb6fa26803c columnFamilyName B 2024-11-26T16:42:41,056 DEBUG [StoreOpener-42f6c759eca28a5c6a511cb6fa26803c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:41,057 INFO [StoreOpener-42f6c759eca28a5c6a511cb6fa26803c-1 {}] regionserver.HStore(327): Store=42f6c759eca28a5c6a511cb6fa26803c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:42:41,057 INFO [StoreOpener-42f6c759eca28a5c6a511cb6fa26803c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:41,057 INFO [StoreOpener-42f6c759eca28a5c6a511cb6fa26803c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:42:41,058 INFO [StoreOpener-42f6c759eca28a5c6a511cb6fa26803c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42f6c759eca28a5c6a511cb6fa26803c columnFamilyName C 2024-11-26T16:42:41,058 DEBUG [StoreOpener-42f6c759eca28a5c6a511cb6fa26803c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:42:41,058 INFO [StoreOpener-42f6c759eca28a5c6a511cb6fa26803c-1 {}] regionserver.HStore(327): Store=42f6c759eca28a5c6a511cb6fa26803c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:42:41,058 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:41,058 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1038): replaying wal for 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:41,059 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:41,059 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:41,060 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1048): stopping wal replay for 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:41,060 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1060): Cleaning up temporary data for 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:41,060 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T16:42:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-26T16:42:41,061 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1093): writing seq id for 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:41,063 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T16:42:41,063 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1114): Opened 42f6c759eca28a5c6a511cb6fa26803c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63304039, jitterRate=-0.05669631063938141}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T16:42:41,063 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:41,064 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1006): Region open journal for 42f6c759eca28a5c6a511cb6fa26803c: Running coprocessor pre-open hook at 1732639361052Writing region info on filesystem at 1732639361052Initializing all the Stores at 1732639361053 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639361053Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639361053Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639361053Cleaning up temporary data from old regions at 1732639361060 (+7 ms)Running coprocessor post-open hooks at 1732639361063 (+3 ms)Region opened successfully at 1732639361064 (+1 ms) 2024-11-26T16:42:41,065 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., pid=110, masterSystemTime=1732639361048 2024-11-26T16:42:41,066 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:41,066 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:41,067 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=42f6c759eca28a5c6a511cb6fa26803c, regionState=OPEN, openSeqNum=2, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:41,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 because future has completed 2024-11-26T16:42:41,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-26T16:42:41,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; OpenRegionProcedure 42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 in 171 msec 2024-11-26T16:42:41,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-26T16:42:41,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42f6c759eca28a5c6a511cb6fa26803c, ASSIGN in 327 msec 2024-11-26T16:42:41,072 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T16:42:41,072 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639361072"}]},"ts":"1732639361072"} 2024-11-26T16:42:41,073 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T16:42:41,074 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T16:42:41,075 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1510 sec 2024-11-26T16:42:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-11-26T16:42:42,071 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:42,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1657d9e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,072 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:42,073 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:42,074 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4260de03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:42,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:42,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:42,076 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:42,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23487433, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,076 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:42,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:42,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:42,078 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:42,079 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T16:42:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 2dedfe7466e8,35233,1732639237322 2024-11-26T16:42:42,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2b0d4095 2024-11-26T16:42:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T16:42:42,080 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T16:42:42,081 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19a3b235, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,081 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:42,081 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:42,082 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@326dd1a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:42,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:42,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:42,083 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:42,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b86f64b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,084 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@720699be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,084 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:42,084 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:42,085 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ae5597d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:42,086 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c37cd37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,087 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78e2c23d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,087 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:42,087 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:42,088 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:42,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:42,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:42,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@490b1cf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:42,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:42,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:42,089 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:42,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5407e391, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,090 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1931bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,090 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:42,090 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:42,091 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2adf5f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:42,092 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a7e63a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,092 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143510a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,093 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:42,093 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:42,093 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:42,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:42,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:42,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@541e0aaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:42,094 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478754d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,095 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e46d68a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,095 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:42,095 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:42,096 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@671d9e97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:42,097 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@293d2336, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,098 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@127790d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,098 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:42,098 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:42,098 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:42,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:42,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:42,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cde1bbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:42,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:42,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:42,099 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:42,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a2cdb4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,100 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2461d367, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,100 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:42,100 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:42,101 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:42,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:42,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:42,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17a131c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:42,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:42,102 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:42,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b43b68f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12a13cd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,103 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:42,103 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:42,103 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65574e3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:42,105 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20134f54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cf53828, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,105 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:42:42,106 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:42:42,106 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:42:42,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:42:42,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:42:42,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554d96b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:42:42,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:42:42,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:42:42,107 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:42:42,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68f4c9f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:42:42,108 DEBUG [Thread-1966 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:42,108 DEBUG [Thread-1949 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:42,108 DEBUG [Thread-1955 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:42,109 DEBUG [Thread-1957 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:42,109 DEBUG [Thread-1964 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:42,109 DEBUG [Thread-1968 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:42,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:42:42,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:42,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:42,110 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:42,110 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:42,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:42,111 DEBUG [Thread-1962 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:42,111 DEBUG [Thread-1960 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-26T16:42:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:42,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:42,112 DEBUG [Thread-1953 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:42,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:42,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:42,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:42:42,113 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:42:42,113 DEBUG [Thread-1951 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:42:42,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T16:42:42,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44042, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:42,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:42,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:42,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:42,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:42,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:42:42,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:42:42,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:42:42,115 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:42:42,115 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:42,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:42:42,115 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:42:42,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:42:42,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:42:42,117 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:42:42,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:42:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:42:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:42:42,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:42:42,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2] 2024-11-26T16:42:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:42,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T16:42:42,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:42:42,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:42,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:42:42,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:42,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:42:42,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:42,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/c644c2638e814089a732700ee200b34e is 50, key is test_row_0/A:col10/1732639362124/Put/seqid=0 2024-11-26T16:42:42,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742258_1434 (size=19021) 2024-11-26T16:42:42,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639422163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:42,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639422164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:42,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639422177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:42,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:42,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639422179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639422180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T16:42:42,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T16:42:42,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:42,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:42,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:42,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:42,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:42,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:42,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:42,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639422291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:42,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:42,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639422291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:42,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:42,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639422300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:42,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639422301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:42,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639422301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:42,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T16:42:42,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:42,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:42,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:42,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:42,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T16:42:42,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639422501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:42,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639422501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:42,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:42,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639422511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:42,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:42,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:42,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639422511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:42,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:42,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639422511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:42,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:42,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:42,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/c644c2638e814089a732700ee200b34e 2024-11-26T16:42:42,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T16:42:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:42,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:42,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/23b8ad53230e4abba9bcb1882b897f00 is 50, key is test_row_0/B:col10/1732639362124/Put/seqid=0 2024-11-26T16:42:42,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742259_1435 (size=12001) 2024-11-26T16:42:42,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/23b8ad53230e4abba9bcb1882b897f00 2024-11-26T16:42:42,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/987ec28dd7754bf3bb652f7761a740ee is 50, key is test_row_0/C:col10/1732639362124/Put/seqid=0 2024-11-26T16:42:42,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742260_1436 (size=12001) 2024-11-26T16:42:42,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T16:42:42,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:42,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:42,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:42,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:42,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:42,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T16:42:42,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639422821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:42,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639422821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:42,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:42,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:42,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639422822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639422822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:42,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639422822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:42,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T16:42:42,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:42,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:42,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:42,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:42,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:42,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:43,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/987ec28dd7754bf3bb652f7761a740ee 2024-11-26T16:42:43,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T16:42:43,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:43,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:43,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:43,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:43,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:43,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/c644c2638e814089a732700ee200b34e as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c644c2638e814089a732700ee200b34e 2024-11-26T16:42:43,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c644c2638e814089a732700ee200b34e, entries=300, sequenceid=14, filesize=18.6 K 2024-11-26T16:42:43,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/23b8ad53230e4abba9bcb1882b897f00 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/23b8ad53230e4abba9bcb1882b897f00 2024-11-26T16:42:43,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/23b8ad53230e4abba9bcb1882b897f00, entries=150, sequenceid=14, filesize=11.7 K 2024-11-26T16:42:43,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/987ec28dd7754bf3bb652f7761a740ee as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/987ec28dd7754bf3bb652f7761a740ee 2024-11-26T16:42:43,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/987ec28dd7754bf3bb652f7761a740ee, entries=150, sequenceid=14, filesize=11.7 K 2024-11-26T16:42:43,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 42f6c759eca28a5c6a511cb6fa26803c in 926ms, sequenceid=14, compaction requested=false 2024-11-26T16:42:43,054 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-26T16:42:43,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:43,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-26T16:42:43,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:43,191 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T16:42:43,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:42:43,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:43,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:42:43,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:43,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:42:43,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:43,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/9889a0c1ebd7454793e3528034a07686 is 50, key is test_row_0/A:col10/1732639362175/Put/seqid=0 2024-11-26T16:42:43,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742261_1437 (size=12001) 2024-11-26T16:42:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T16:42:43,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:43,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:43,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:43,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639423351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:43,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:43,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:43,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639423360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:43,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:43,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639423361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:43,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:43,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:43,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639423363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:43,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:43,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639423363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:43,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:43,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:43,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:43,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639423471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:43,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:43,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639423471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:43,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:43,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:43,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639423481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639423482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:43,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639423482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:43,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:43,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:43,601 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/9889a0c1ebd7454793e3528034a07686 2024-11-26T16:42:43,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/b0fca830eeb64caea886a31105c06832 is 50, key is test_row_0/B:col10/1732639362175/Put/seqid=0 2024-11-26T16:42:43,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742262_1438 (size=12001) 2024-11-26T16:42:43,613 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/b0fca830eeb64caea886a31105c06832 2024-11-26T16:42:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/75d6789ffe7c40f3b9e29eb385a8ced9 is 50, key is test_row_0/C:col10/1732639362175/Put/seqid=0 2024-11-26T16:42:43,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742263_1439 (size=12001) 2024-11-26T16:42:43,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639423691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639423692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:43,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639423701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:43,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639423701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:43,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639423701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:43,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:44,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639424001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:44,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639424001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:44,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639424011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:44,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:44,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639424011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:44,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:44,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639424021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:44,026 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/75d6789ffe7c40f3b9e29eb385a8ced9 2024-11-26T16:42:44,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/9889a0c1ebd7454793e3528034a07686 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/9889a0c1ebd7454793e3528034a07686 2024-11-26T16:42:44,035 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/9889a0c1ebd7454793e3528034a07686, entries=150, sequenceid=33, filesize=11.7 K 2024-11-26T16:42:44,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/b0fca830eeb64caea886a31105c06832 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/b0fca830eeb64caea886a31105c06832 2024-11-26T16:42:44,040 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/b0fca830eeb64caea886a31105c06832, entries=150, sequenceid=33, filesize=11.7 K 2024-11-26T16:42:44,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/75d6789ffe7c40f3b9e29eb385a8ced9 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/75d6789ffe7c40f3b9e29eb385a8ced9 2024-11-26T16:42:44,044 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/75d6789ffe7c40f3b9e29eb385a8ced9, entries=150, sequenceid=33, filesize=11.7 K 2024-11-26T16:42:44,045 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 42f6c759eca28a5c6a511cb6fa26803c in 854ms, sequenceid=33, compaction requested=false 2024-11-26T16:42:44,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:44,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:44,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-26T16:42:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-26T16:42:44,048 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-26T16:42:44,048 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9310 sec 2024-11-26T16:42:44,050 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.9400 sec 2024-11-26T16:42:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-26T16:42:44,262 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:44,262 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:42:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-26T16:42:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T16:42:44,264 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:42:44,265 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:42:44,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:42:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T16:42:44,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-26T16:42:44,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:44,417 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T16:42:44,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:42:44,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:44,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:42:44,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:44,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:42:44,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:44,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/e00883caa2384888befef8042ed6a636 is 50, key is test_row_1/A:col10/1732639363358/Put/seqid=0 2024-11-26T16:42:44,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742264_1440 (size=9657) 2024-11-26T16:42:44,428 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/e00883caa2384888befef8042ed6a636 2024-11-26T16:42:44,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/06a23e3b6bb04027bb4434046ceaceaa is 50, key is test_row_1/B:col10/1732639363358/Put/seqid=0 2024-11-26T16:42:44,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742265_1441 (size=9657) 2024-11-26T16:42:44,440 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/06a23e3b6bb04027bb4434046ceaceaa 2024-11-26T16:42:44,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/56992abf7728400a8c888d7e37ec1aea is 50, key is test_row_1/C:col10/1732639363358/Put/seqid=0 2024-11-26T16:42:44,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742266_1442 (size=9657) 2024-11-26T16:42:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:44,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:44,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639424555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:44,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639424556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:44,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:44,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:44,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639424557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:44,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:44,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639424569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:44,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:44,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639424569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T16:42:44,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639424671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:44,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:44,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639424671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:44,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:44,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:44,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639424671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:44,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:44,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:44,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639424681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:44,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639424681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:44,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:44,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:44,860 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/56992abf7728400a8c888d7e37ec1aea 2024-11-26T16:42:44,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/e00883caa2384888befef8042ed6a636 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/e00883caa2384888befef8042ed6a636 2024-11-26T16:42:44,868 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/e00883caa2384888befef8042ed6a636, entries=100, sequenceid=46, filesize=9.4 K 2024-11-26T16:42:44,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/06a23e3b6bb04027bb4434046ceaceaa as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/06a23e3b6bb04027bb4434046ceaceaa 2024-11-26T16:42:44,872 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/06a23e3b6bb04027bb4434046ceaceaa, entries=100, sequenceid=46, filesize=9.4 K 2024-11-26T16:42:44,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/56992abf7728400a8c888d7e37ec1aea as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/56992abf7728400a8c888d7e37ec1aea 2024-11-26T16:42:44,876 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/56992abf7728400a8c888d7e37ec1aea, entries=100, sequenceid=46, filesize=9.4 K 2024-11-26T16:42:44,877 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 42f6c759eca28a5c6a511cb6fa26803c in 460ms, sequenceid=46, compaction requested=true 2024-11-26T16:42:44,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:44,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:44,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-26T16:42:44,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-26T16:42:44,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-26T16:42:44,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 614 msec 2024-11-26T16:42:44,882 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 619 msec 2024-11-26T16:42:44,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:44,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-26T16:42:44,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:42:44,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:44,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:42:44,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:44,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:42:44,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:44,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-26T16:42:44,892 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:44,892 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:42:44,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-26T16:42:44,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T16:42:44,894 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:42:44,894 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:42:44,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:42:44,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/90aaf67a7bc048ef9d6f5bac4f124667 is 50, key is test_row_0/A:col10/1732639364555/Put/seqid=0 2024-11-26T16:42:44,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742267_1443 (size=14341) 2024-11-26T16:42:44,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:44,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639424940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:44,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639424941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:44,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:44,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639424942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:44,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639424943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:44,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639424945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:45,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T16:42:45,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T16:42:45,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:45,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:45,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:45,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:45,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:45,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:45,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:45,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639425051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:45,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639425052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:45,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:45,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639425071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:45,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:45,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639425071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:45,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:45,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:45,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639425071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:45,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:45,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:45,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T16:42:45,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:45,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:45,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:45,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:45,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:45,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:45,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T16:42:45,248 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T16:42:45,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639425271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:45,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639425271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:45,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:45,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639425281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:45,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:45,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639425291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:45,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/90aaf67a7bc048ef9d6f5bac4f124667 2024-11-26T16:42:45,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:45,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639425292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:45,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:45,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:45,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/2a2d906fb66841d0a3d9dedbb2d67048 is 50, key is test_row_0/B:col10/1732639364555/Put/seqid=0 2024-11-26T16:42:45,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742268_1444 (size=12001) 2024-11-26T16:42:45,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T16:42:45,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:45,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:45,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:45,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:45,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:45,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T16:42:45,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:45,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:45,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:45,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:45,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:45,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T16:42:45,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639425591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:45,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639425591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:45,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639425601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:45,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:45,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639425610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:45,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639425610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:45,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T16:42:45,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:45,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:45,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:45,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:45,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:45,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/2a2d906fb66841d0a3d9dedbb2d67048 2024-11-26T16:42:45,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/344315cc63da4c58ac0f46fde313bc9a is 50, key is test_row_0/C:col10/1732639364555/Put/seqid=0 2024-11-26T16:42:45,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742269_1445 (size=12001) 2024-11-26T16:42:45,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T16:42:45,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:45,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:45,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:45,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:45,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:45,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T16:42:45,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:45,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:45,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:45,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:45,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:46,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T16:42:46,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T16:42:46,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:46,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:46,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:46,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:46,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:46,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639426111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:46,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:46,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639426111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:46,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:46,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639426111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:46,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:46,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/344315cc63da4c58ac0f46fde313bc9a 2024-11-26T16:42:46,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/90aaf67a7bc048ef9d6f5bac4f124667 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/90aaf67a7bc048ef9d6f5bac4f124667 2024-11-26T16:42:46,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/90aaf67a7bc048ef9d6f5bac4f124667, entries=200, sequenceid=67, filesize=14.0 K 2024-11-26T16:42:46,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/2a2d906fb66841d0a3d9dedbb2d67048 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/2a2d906fb66841d0a3d9dedbb2d67048 2024-11-26T16:42:46,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/2a2d906fb66841d0a3d9dedbb2d67048, entries=150, sequenceid=67, filesize=11.7 K 2024-11-26T16:42:46,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/344315cc63da4c58ac0f46fde313bc9a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/344315cc63da4c58ac0f46fde313bc9a 2024-11-26T16:42:46,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/344315cc63da4c58ac0f46fde313bc9a, entries=150, sequenceid=67, filesize=11.7 K 2024-11-26T16:42:46,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 42f6c759eca28a5c6a511cb6fa26803c in 1253ms, sequenceid=67, compaction requested=true 2024-11-26T16:42:46,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:46,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:42:46,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:46,142 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:46,142 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:46,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:42:46,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:46,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:46,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:46,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T16:42:46,144 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:46,144 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55020 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:46,144 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/B is initiating minor compaction (all files) 2024-11-26T16:42:46,144 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/A is initiating minor compaction (all files) 2024-11-26T16:42:46,144 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/A in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:46,144 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/B in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:46,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:42:46,144 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/23b8ad53230e4abba9bcb1882b897f00, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/b0fca830eeb64caea886a31105c06832, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/06a23e3b6bb04027bb4434046ceaceaa, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/2a2d906fb66841d0a3d9dedbb2d67048] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=44.6 K 2024-11-26T16:42:46,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:46,144 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c644c2638e814089a732700ee200b34e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/9889a0c1ebd7454793e3528034a07686, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/e00883caa2384888befef8042ed6a636, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/90aaf67a7bc048ef9d6f5bac4f124667] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=53.7 K 2024-11-26T16:42:46,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:42:46,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:46,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:42:46,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:46,145 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 23b8ad53230e4abba9bcb1882b897f00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732639362124 2024-11-26T16:42:46,145 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting c644c2638e814089a732700ee200b34e, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732639362114 2024-11-26T16:42:46,146 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting b0fca830eeb64caea886a31105c06832, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732639362175 2024-11-26T16:42:46,146 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9889a0c1ebd7454793e3528034a07686, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732639362175 2024-11-26T16:42:46,146 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 06a23e3b6bb04027bb4434046ceaceaa, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732639363358 2024-11-26T16:42:46,146 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting e00883caa2384888befef8042ed6a636, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732639363358 2024-11-26T16:42:46,147 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a2d906fb66841d0a3d9dedbb2d67048, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732639364555 2024-11-26T16:42:46,147 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90aaf67a7bc048ef9d6f5bac4f124667, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732639364554 2024-11-26T16:42:46,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/55ba2e99fd1f46e29ed2c8a1db33b81d is 50, key is test_row_0/A:col10/1732639364943/Put/seqid=0 2024-11-26T16:42:46,165 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#A#compaction#373 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:46,165 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/a056b30325c94a7c883300f8ffbb779b is 50, key is test_row_0/A:col10/1732639364555/Put/seqid=0 2024-11-26T16:42:46,166 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#B#compaction#374 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:46,166 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/0720ef26e1504bf08d714a88880d55ff is 50, key is test_row_0/B:col10/1732639364555/Put/seqid=0 2024-11-26T16:42:46,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742270_1446 (size=14341) 2024-11-26T16:42:46,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742272_1448 (size=12139) 2024-11-26T16:42:46,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742271_1447 (size=12139) 2024-11-26T16:42:46,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639426244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:46,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:46,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639426253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:46,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:46,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T16:42:46,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:46,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:46,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:46,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:46,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:46,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639426361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:46,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:46,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639426371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:46,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T16:42:46,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:46,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:46,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:46,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:46,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:46,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/55ba2e99fd1f46e29ed2c8a1db33b81d 2024-11-26T16:42:46,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/035e1139026e42a3ba8d02dc912b8f98 is 50, key is test_row_0/B:col10/1732639364943/Put/seqid=0 2024-11-26T16:42:46,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T16:42:46,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:46,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:46,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:46,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:46,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:46,579 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/0720ef26e1504bf08d714a88880d55ff as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/0720ef26e1504bf08d714a88880d55ff 2024-11-26T16:42:46,584 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/a056b30325c94a7c883300f8ffbb779b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a056b30325c94a7c883300f8ffbb779b 2024-11-26T16:42:46,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639426581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:46,586 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/B of 42f6c759eca28a5c6a511cb6fa26803c into 0720ef26e1504bf08d714a88880d55ff(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:46,586 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,586 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/B, priority=12, startTime=1732639366142; duration=0sec 2024-11-26T16:42:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639426581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:46,586 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:46,586 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:B 2024-11-26T16:42:46,586 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:46,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742273_1449 (size=12001) 2024-11-26T16:42:46,590 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:46,590 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/C is initiating minor compaction (all files) 2024-11-26T16:42:46,590 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/C in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:46,590 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/987ec28dd7754bf3bb652f7761a740ee, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/75d6789ffe7c40f3b9e29eb385a8ced9, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/56992abf7728400a8c888d7e37ec1aea, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/344315cc63da4c58ac0f46fde313bc9a] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=44.6 K 2024-11-26T16:42:46,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/035e1139026e42a3ba8d02dc912b8f98 2024-11-26T16:42:46,591 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 987ec28dd7754bf3bb652f7761a740ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732639362124 2024-11-26T16:42:46,591 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 75d6789ffe7c40f3b9e29eb385a8ced9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732639362175 2024-11-26T16:42:46,591 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/A of 42f6c759eca28a5c6a511cb6fa26803c into a056b30325c94a7c883300f8ffbb779b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:46,591 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:46,591 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 56992abf7728400a8c888d7e37ec1aea, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732639363358 2024-11-26T16:42:46,591 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/A, priority=12, startTime=1732639366142; duration=0sec 2024-11-26T16:42:46,592 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:46,592 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:A 2024-11-26T16:42:46,592 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 344315cc63da4c58ac0f46fde313bc9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732639364555 2024-11-26T16:42:46,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/0b5898f264e44d80bce0ce389dc751e7 is 50, key is test_row_0/C:col10/1732639364943/Put/seqid=0 2024-11-26T16:42:46,601 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#C#compaction#377 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:46,602 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/05cbb6de2f2c4701a14b18bb65172ae8 is 50, key is test_row_0/C:col10/1732639364555/Put/seqid=0 2024-11-26T16:42:46,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742274_1450 (size=12001) 2024-11-26T16:42:46,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742275_1451 (size=12139) 2024-11-26T16:42:46,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T16:42:46,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:46,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:46,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:46,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:46,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:46,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T16:42:46,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:46,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:46,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:46,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:46,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:46,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:46,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639426891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:46,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639426891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:47,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/0b5898f264e44d80bce0ce389dc751e7 2024-11-26T16:42:47,016 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/05cbb6de2f2c4701a14b18bb65172ae8 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/05cbb6de2f2c4701a14b18bb65172ae8 2024-11-26T16:42:47,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/55ba2e99fd1f46e29ed2c8a1db33b81d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/55ba2e99fd1f46e29ed2c8a1db33b81d 2024-11-26T16:42:47,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/55ba2e99fd1f46e29ed2c8a1db33b81d, entries=200, sequenceid=79, filesize=14.0 K 2024-11-26T16:42:47,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/035e1139026e42a3ba8d02dc912b8f98 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/035e1139026e42a3ba8d02dc912b8f98 2024-11-26T16:42:47,021 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/C of 42f6c759eca28a5c6a511cb6fa26803c into 05cbb6de2f2c4701a14b18bb65172ae8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:47,021 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:47,021 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/C, priority=12, startTime=1732639366142; duration=0sec 2024-11-26T16:42:47,021 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:47,021 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:C 2024-11-26T16:42:47,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/035e1139026e42a3ba8d02dc912b8f98, entries=150, sequenceid=79, filesize=11.7 K 2024-11-26T16:42:47,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/0b5898f264e44d80bce0ce389dc751e7 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/0b5898f264e44d80bce0ce389dc751e7 2024-11-26T16:42:47,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/0b5898f264e44d80bce0ce389dc751e7, entries=150, sequenceid=79, filesize=11.7 K 2024-11-26T16:42:47,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 42f6c759eca28a5c6a511cb6fa26803c in 889ms, sequenceid=79, compaction requested=false 2024-11-26T16:42:47,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:47,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-26T16:42:47,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:47,037 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T16:42:47,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:42:47,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:47,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:42:47,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:47,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:42:47,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:47,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T16:42:47,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/b7f28d35561d4a168279864e62acdfc2 is 50, key is test_row_0/A:col10/1732639366211/Put/seqid=0 2024-11-26T16:42:47,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742276_1452 (size=12001) 2024-11-26T16:42:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:47,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:47,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:47,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639427165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:47,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:47,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:47,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639427165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:47,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:47,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639427166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:47,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:47,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:47,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639427271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:47,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:47,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639427271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:47,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:47,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639427291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:47,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:47,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639427411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:47,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:47,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639427411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:47,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:47,446 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/b7f28d35561d4a168279864e62acdfc2 2024-11-26T16:42:47,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/f3b42eeb55b5492889bfc3c5427d681f is 50, key is test_row_0/B:col10/1732639366211/Put/seqid=0 2024-11-26T16:42:47,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742277_1453 (size=12001) 2024-11-26T16:42:47,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639427481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:47,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:47,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639427481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:47,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639427511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:47,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639427791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:47,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:47,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639427801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:47,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639427821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:47,858 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/f3b42eeb55b5492889bfc3c5427d681f 2024-11-26T16:42:47,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/a3aaa8637e58488faf5368c6522feb09 is 50, key is test_row_0/C:col10/1732639366211/Put/seqid=0 2024-11-26T16:42:47,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742278_1454 (size=12001) 2024-11-26T16:42:48,270 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/a3aaa8637e58488faf5368c6522feb09 2024-11-26T16:42:48,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/b7f28d35561d4a168279864e62acdfc2 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/b7f28d35561d4a168279864e62acdfc2 2024-11-26T16:42:48,277 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/b7f28d35561d4a168279864e62acdfc2, entries=150, sequenceid=102, filesize=11.7 K 2024-11-26T16:42:48,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/f3b42eeb55b5492889bfc3c5427d681f as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/f3b42eeb55b5492889bfc3c5427d681f 2024-11-26T16:42:48,282 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/f3b42eeb55b5492889bfc3c5427d681f, entries=150, sequenceid=102, filesize=11.7 K 2024-11-26T16:42:48,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/a3aaa8637e58488faf5368c6522feb09 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a3aaa8637e58488faf5368c6522feb09 2024-11-26T16:42:48,286 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a3aaa8637e58488faf5368c6522feb09, entries=150, sequenceid=102, filesize=11.7 K 2024-11-26T16:42:48,287 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 42f6c759eca28a5c6a511cb6fa26803c in 1250ms, sequenceid=102, compaction requested=true 2024-11-26T16:42:48,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:48,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:48,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-26T16:42:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-26T16:42:48,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-26T16:42:48,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3930 sec 2024-11-26T16:42:48,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 3.3990 sec 2024-11-26T16:42:48,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:48,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-26T16:42:48,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:42:48,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:48,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:42:48,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:48,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:42:48,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:48,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/a6db0da0f0a74bc8b57eabf4749937ed is 50, key is test_row_0/A:col10/1732639367148/Put/seqid=0 2024-11-26T16:42:48,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742279_1455 (size=14341) 2024-11-26T16:42:48,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639428386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:48,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:48,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639428387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:48,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639428388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:48,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639428421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:48,431 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in 42f6c759eca28a5c6a511cb6fa26803c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:48,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:48,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639428431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:48,441 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in 42f6c759eca28a5c6a511cb6fa26803c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:48,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:48,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639428501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:48,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639428511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:48,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:48,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639428511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:48,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:48,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:48,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/a6db0da0f0a74bc8b57eabf4749937ed 2024-11-26T16:42:48,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:48,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639428721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:48,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/1fec3270cbba45358c02922717b6b41e is 50, key is test_row_0/B:col10/1732639367148/Put/seqid=0 2024-11-26T16:42:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:48,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639428721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:48,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639428721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:48,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:48,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:48,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742280_1456 (size=12001) 2024-11-26T16:42:49,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:49,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639429041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:49,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639429041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:49,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639429041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-26T16:42:49,061 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:49,062 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:42:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-26T16:42:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T16:42:49,064 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:42:49,065 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:42:49,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:42:49,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/1fec3270cbba45358c02922717b6b41e 2024-11-26T16:42:49,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/d196a73e15f847cbac1b321bfc533c9a is 50, key is test_row_0/C:col10/1732639367148/Put/seqid=0 2024-11-26T16:42:49,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742281_1457 (size=12001) 2024-11-26T16:42:49,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/d196a73e15f847cbac1b321bfc533c9a 2024-11-26T16:42:49,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/a6db0da0f0a74bc8b57eabf4749937ed as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a6db0da0f0a74bc8b57eabf4749937ed 2024-11-26T16:42:49,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a6db0da0f0a74bc8b57eabf4749937ed, entries=200, sequenceid=115, filesize=14.0 K 2024-11-26T16:42:49,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/1fec3270cbba45358c02922717b6b41e as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/1fec3270cbba45358c02922717b6b41e 2024-11-26T16:42:49,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/1fec3270cbba45358c02922717b6b41e, entries=150, sequenceid=115, filesize=11.7 K 2024-11-26T16:42:49,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/d196a73e15f847cbac1b321bfc533c9a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/d196a73e15f847cbac1b321bfc533c9a 2024-11-26T16:42:49,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/d196a73e15f847cbac1b321bfc533c9a, entries=150, sequenceid=115, filesize=11.7 K 2024-11-26T16:42:49,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 42f6c759eca28a5c6a511cb6fa26803c in 855ms, sequenceid=115, compaction requested=true 2024-11-26T16:42:49,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T16:42:49,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:49,171 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:49,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:42:49,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:49,172 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:49,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:42:49,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:49,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:49,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:49,173 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52822 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:49,173 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/A is initiating minor compaction (all files) 2024-11-26T16:42:49,173 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/A in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:49,173 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a056b30325c94a7c883300f8ffbb779b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/55ba2e99fd1f46e29ed2c8a1db33b81d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/b7f28d35561d4a168279864e62acdfc2, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a6db0da0f0a74bc8b57eabf4749937ed] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=51.6 K 2024-11-26T16:42:49,174 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting a056b30325c94a7c883300f8ffbb779b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732639364555 2024-11-26T16:42:49,175 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55ba2e99fd1f46e29ed2c8a1db33b81d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732639364938 2024-11-26T16:42:49,175 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:49,175 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7f28d35561d4a168279864e62acdfc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732639366211 2024-11-26T16:42:49,175 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/B is initiating minor compaction (all files) 2024-11-26T16:42:49,175 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/B in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:49,175 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/0720ef26e1504bf08d714a88880d55ff, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/035e1139026e42a3ba8d02dc912b8f98, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/f3b42eeb55b5492889bfc3c5427d681f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/1fec3270cbba45358c02922717b6b41e] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=47.0 K 2024-11-26T16:42:49,176 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6db0da0f0a74bc8b57eabf4749937ed, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732639367148 2024-11-26T16:42:49,176 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 0720ef26e1504bf08d714a88880d55ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732639364555 2024-11-26T16:42:49,176 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 035e1139026e42a3ba8d02dc912b8f98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732639364940 2024-11-26T16:42:49,177 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting f3b42eeb55b5492889bfc3c5427d681f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732639366211 2024-11-26T16:42:49,177 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fec3270cbba45358c02922717b6b41e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732639367148 2024-11-26T16:42:49,188 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#A#compaction#384 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:49,188 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#B#compaction#385 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:49,188 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/a79723cf49ad4a5ba7c6923e92a7a3b8 is 50, key is test_row_0/A:col10/1732639367148/Put/seqid=0 2024-11-26T16:42:49,188 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/6ca83f4de45e4079b6f7a1ffce4d7498 is 50, key is test_row_0/B:col10/1732639367148/Put/seqid=0 2024-11-26T16:42:49,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742282_1458 (size=12275) 2024-11-26T16:42:49,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742283_1459 (size=12275) 2024-11-26T16:42:49,205 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/6ca83f4de45e4079b6f7a1ffce4d7498 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/6ca83f4de45e4079b6f7a1ffce4d7498 2024-11-26T16:42:49,211 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/B of 42f6c759eca28a5c6a511cb6fa26803c into 6ca83f4de45e4079b6f7a1ffce4d7498(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:49,211 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:49,211 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/B, priority=12, startTime=1732639369172; duration=0sec 2024-11-26T16:42:49,212 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:49,212 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:B 2024-11-26T16:42:49,212 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:49,213 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:49,213 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/C is initiating minor compaction (all files) 2024-11-26T16:42:49,213 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/C in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:49,213 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/05cbb6de2f2c4701a14b18bb65172ae8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/0b5898f264e44d80bce0ce389dc751e7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a3aaa8637e58488faf5368c6522feb09, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/d196a73e15f847cbac1b321bfc533c9a] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=47.0 K 2024-11-26T16:42:49,214 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 05cbb6de2f2c4701a14b18bb65172ae8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732639364555 2024-11-26T16:42:49,214 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b5898f264e44d80bce0ce389dc751e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732639364940 2024-11-26T16:42:49,215 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a3aaa8637e58488faf5368c6522feb09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732639366211 2024-11-26T16:42:49,215 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting d196a73e15f847cbac1b321bfc533c9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732639367148 2024-11-26T16:42:49,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-26T16:42:49,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:49,217 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-26T16:42:49,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:42:49,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:49,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:42:49,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:49,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:42:49,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:49,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/7093fbf8ad474107b6ddaa1f859c8a0e is 50, key is test_row_0/A:col10/1732639368384/Put/seqid=0 2024-11-26T16:42:49,228 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#C#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:49,229 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/8f5583bccc7a4f20b6310ef7d4ded3fa is 50, key is test_row_0/C:col10/1732639367148/Put/seqid=0 2024-11-26T16:42:49,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742284_1460 (size=12151) 2024-11-26T16:42:49,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742285_1461 (size=12275) 2024-11-26T16:42:49,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T16:42:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:49,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:49,597 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/a79723cf49ad4a5ba7c6923e92a7a3b8 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a79723cf49ad4a5ba7c6923e92a7a3b8 2024-11-26T16:42:49,602 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/A of 42f6c759eca28a5c6a511cb6fa26803c into a79723cf49ad4a5ba7c6923e92a7a3b8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:49,603 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:49,603 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/A, priority=12, startTime=1732639369171; duration=0sec 2024-11-26T16:42:49,603 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:49,603 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:A 2024-11-26T16:42:49,634 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/7093fbf8ad474107b6ddaa1f859c8a0e 2024-11-26T16:42:49,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639429625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:49,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:49,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639429639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:49,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:49,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639429640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:49,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:49,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/77d6596d57584ae393cd5966f04141bd is 50, key is test_row_0/B:col10/1732639368384/Put/seqid=0 2024-11-26T16:42:49,647 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/8f5583bccc7a4f20b6310ef7d4ded3fa as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/8f5583bccc7a4f20b6310ef7d4ded3fa 2024-11-26T16:42:49,652 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/C of 42f6c759eca28a5c6a511cb6fa26803c into 8f5583bccc7a4f20b6310ef7d4ded3fa(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:49,652 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:49,652 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/C, priority=12, startTime=1732639369172; duration=0sec 2024-11-26T16:42:49,652 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:49,652 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:C 2024-11-26T16:42:49,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742286_1462 (size=12151) 2024-11-26T16:42:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T16:42:49,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639429751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:49,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639429751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:49,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639429751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:49,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639429971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:49,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639429971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:49,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639429971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:49,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:49,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:49,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:50,062 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/77d6596d57584ae393cd5966f04141bd 2024-11-26T16:42:50,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/259dedb66e2b419b82d983866a375231 is 50, key is test_row_0/C:col10/1732639368384/Put/seqid=0 2024-11-26T16:42:50,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742287_1463 (size=12151) 2024-11-26T16:42:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T16:42:50,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:50,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639430281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:50,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:50,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:50,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639430281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:50,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639430281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:50,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:50,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639430441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:50,444 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in 42f6c759eca28a5c6a511cb6fa26803c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:50,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639430461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:50,467 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in 42f6c759eca28a5c6a511cb6fa26803c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:50,473 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/259dedb66e2b419b82d983866a375231 2024-11-26T16:42:50,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/7093fbf8ad474107b6ddaa1f859c8a0e as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/7093fbf8ad474107b6ddaa1f859c8a0e 2024-11-26T16:42:50,480 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/7093fbf8ad474107b6ddaa1f859c8a0e, entries=150, sequenceid=135, filesize=11.9 K 2024-11-26T16:42:50,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/77d6596d57584ae393cd5966f04141bd as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/77d6596d57584ae393cd5966f04141bd 2024-11-26T16:42:50,484 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/77d6596d57584ae393cd5966f04141bd, entries=150, sequenceid=135, filesize=11.9 K 2024-11-26T16:42:50,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/259dedb66e2b419b82d983866a375231 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/259dedb66e2b419b82d983866a375231 2024-11-26T16:42:50,488 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/259dedb66e2b419b82d983866a375231, entries=150, sequenceid=135, filesize=11.9 K 2024-11-26T16:42:50,489 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 42f6c759eca28a5c6a511cb6fa26803c in 1272ms, sequenceid=135, compaction requested=false 2024-11-26T16:42:50,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:50,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:50,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-26T16:42:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-26T16:42:50,492 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-26T16:42:50,492 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4260 sec 2024-11-26T16:42:50,494 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.4310 sec 2024-11-26T16:42:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:50,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T16:42:50,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:42:50,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:50,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:42:50,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:50,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:42:50,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:50,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/6392ee7993ca407d9f979715ad7bff9e is 50, key is test_row_0/A:col10/1732639369638/Put/seqid=0 2024-11-26T16:42:50,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742288_1464 (size=14541) 2024-11-26T16:42:50,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/6392ee7993ca407d9f979715ad7bff9e 2024-11-26T16:42:50,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/458f759690414d7d8a3d400e4c1b4484 is 50, key is test_row_0/B:col10/1732639369638/Put/seqid=0 2024-11-26T16:42:50,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742289_1465 (size=12151) 2024-11-26T16:42:50,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639430856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:50,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639430857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:50,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:50,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639430858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:50,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:50,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639430961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:50,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:50,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639430971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:50,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:50,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639430981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:50,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:50,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:51,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639431181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:51,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:51,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:51,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:51,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639431191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:51,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639431191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:51,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-26T16:42:51,211 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:51,212 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:42:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-26T16:42:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T16:42:51,214 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:42:51,214 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:42:51,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:42:51,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/458f759690414d7d8a3d400e4c1b4484 2024-11-26T16:42:51,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/0537a21c170b4f75b387637639709df9 is 50, key is test_row_0/C:col10/1732639369638/Put/seqid=0 2024-11-26T16:42:51,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742290_1466 (size=12151) 2024-11-26T16:42:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T16:42:51,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T16:42:51,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:51,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:51,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:51,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:51,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:51,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639431491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:51,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:51,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:51,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:51,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T16:42:51,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:51,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:51,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:51,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639431510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:51,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:51,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:51,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:51,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:51,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639431511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:51,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:51,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:51,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:51,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T16:42:51,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/0537a21c170b4f75b387637639709df9 2024-11-26T16:42:51,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/6392ee7993ca407d9f979715ad7bff9e as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/6392ee7993ca407d9f979715ad7bff9e 2024-11-26T16:42:51,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/6392ee7993ca407d9f979715ad7bff9e, entries=200, sequenceid=152, filesize=14.2 K 2024-11-26T16:42:51,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/458f759690414d7d8a3d400e4c1b4484 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/458f759690414d7d8a3d400e4c1b4484 2024-11-26T16:42:51,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/458f759690414d7d8a3d400e4c1b4484, entries=150, sequenceid=152, filesize=11.9 K 2024-11-26T16:42:51,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/0537a21c170b4f75b387637639709df9 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/0537a21c170b4f75b387637639709df9 2024-11-26T16:42:51,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/0537a21c170b4f75b387637639709df9, entries=150, sequenceid=152, filesize=11.9 K 2024-11-26T16:42:51,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 42f6c759eca28a5c6a511cb6fa26803c in 852ms, sequenceid=152, compaction requested=true 2024-11-26T16:42:51,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:51,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:42:51,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:51,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:42:51,659 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:51,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:51,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:51,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:51,659 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:51,660 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:51,660 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38967 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:51,660 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/A is initiating minor compaction (all files) 2024-11-26T16:42:51,660 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/B is initiating minor compaction (all files) 2024-11-26T16:42:51,660 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/A in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:51,660 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/B in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:51,660 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a79723cf49ad4a5ba7c6923e92a7a3b8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/7093fbf8ad474107b6ddaa1f859c8a0e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/6392ee7993ca407d9f979715ad7bff9e] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=38.1 K 2024-11-26T16:42:51,660 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/6ca83f4de45e4079b6f7a1ffce4d7498, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/77d6596d57584ae393cd5966f04141bd, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/458f759690414d7d8a3d400e4c1b4484] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=35.7 K 2024-11-26T16:42:51,660 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting a79723cf49ad4a5ba7c6923e92a7a3b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732639367148 2024-11-26T16:42:51,660 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ca83f4de45e4079b6f7a1ffce4d7498, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732639367148 2024-11-26T16:42:51,661 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 77d6596d57584ae393cd5966f04141bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732639368383 2024-11-26T16:42:51,661 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7093fbf8ad474107b6ddaa1f859c8a0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732639368383 2024-11-26T16:42:51,661 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 458f759690414d7d8a3d400e4c1b4484, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732639369637 2024-11-26T16:42:51,664 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6392ee7993ca407d9f979715ad7bff9e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732639369623 2024-11-26T16:42:51,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-26T16:42:51,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:51,674 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-26T16:42:51,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:42:51,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:51,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:42:51,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:51,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:42:51,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:51,676 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#A#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:51,676 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#B#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:51,677 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/c8e971330edf4002ab8a501220c06f21 is 50, key is test_row_0/B:col10/1732639369638/Put/seqid=0 2024-11-26T16:42:51,677 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/17e46581743f484687fd0200712d0069 is 50, key is test_row_0/A:col10/1732639369638/Put/seqid=0 2024-11-26T16:42:51,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/7c585aa0d209472484be99b1ca2aaa80 is 50, key is test_row_0/A:col10/1732639370842/Put/seqid=0 2024-11-26T16:42:51,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742291_1467 (size=12527) 2024-11-26T16:42:51,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742292_1468 (size=12527) 2024-11-26T16:42:51,698 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/c8e971330edf4002ab8a501220c06f21 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/c8e971330edf4002ab8a501220c06f21 2024-11-26T16:42:51,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742293_1469 (size=12151) 2024-11-26T16:42:51,706 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/B of 42f6c759eca28a5c6a511cb6fa26803c into c8e971330edf4002ab8a501220c06f21(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:51,706 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:51,706 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/B, priority=13, startTime=1732639371658; duration=0sec 2024-11-26T16:42:51,706 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:51,706 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:B 2024-11-26T16:42:51,706 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:51,708 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/7c585aa0d209472484be99b1ca2aaa80 2024-11-26T16:42:51,708 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:51,708 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/C is initiating minor compaction (all files) 2024-11-26T16:42:51,708 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/C in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:51,708 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/8f5583bccc7a4f20b6310ef7d4ded3fa, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/259dedb66e2b419b82d983866a375231, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/0537a21c170b4f75b387637639709df9] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=35.7 K 2024-11-26T16:42:51,709 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f5583bccc7a4f20b6310ef7d4ded3fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732639367148 2024-11-26T16:42:51,713 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 259dedb66e2b419b82d983866a375231, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732639368383 2024-11-26T16:42:51,716 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 0537a21c170b4f75b387637639709df9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732639369637 2024-11-26T16:42:51,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/4378acfdc44945fb956538b1b95629aa is 50, key is test_row_0/B:col10/1732639370842/Put/seqid=0 2024-11-26T16:42:51,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742294_1470 (size=12151) 2024-11-26T16:42:51,732 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/4378acfdc44945fb956538b1b95629aa 2024-11-26T16:42:51,743 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#C#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:51,743 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/5c4e5ce368b748a4aa728da71e82f90b is 50, key is test_row_0/C:col10/1732639369638/Put/seqid=0 2024-11-26T16:42:51,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/117a7871cadd4d04a906aa56659682d5 is 50, key is test_row_0/C:col10/1732639370842/Put/seqid=0 2024-11-26T16:42:51,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742295_1471 (size=12527) 2024-11-26T16:42:51,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742296_1472 (size=12151) 2024-11-26T16:42:51,762 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/117a7871cadd4d04a906aa56659682d5 2024-11-26T16:42:51,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/7c585aa0d209472484be99b1ca2aaa80 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/7c585aa0d209472484be99b1ca2aaa80 2024-11-26T16:42:51,772 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/7c585aa0d209472484be99b1ca2aaa80, entries=150, sequenceid=170, filesize=11.9 K 2024-11-26T16:42:51,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/4378acfdc44945fb956538b1b95629aa as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/4378acfdc44945fb956538b1b95629aa 2024-11-26T16:42:51,776 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/4378acfdc44945fb956538b1b95629aa, entries=150, sequenceid=170, filesize=11.9 K 2024-11-26T16:42:51,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/117a7871cadd4d04a906aa56659682d5 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/117a7871cadd4d04a906aa56659682d5 2024-11-26T16:42:51,781 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/117a7871cadd4d04a906aa56659682d5, entries=150, sequenceid=170, filesize=11.9 K 2024-11-26T16:42:51,782 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 42f6c759eca28a5c6a511cb6fa26803c in 108ms, sequenceid=170, compaction requested=false 2024-11-26T16:42:51,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:51,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:51,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-26T16:42:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-26T16:42:51,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-26T16:42:51,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 570 msec 2024-11-26T16:42:51,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 576 msec 2024-11-26T16:42:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-26T16:42:51,841 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:51,841 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:42:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-26T16:42:51,843 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:42:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T16:42:51,844 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:42:51,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:42:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T16:42:51,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-26T16:42:51,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:51,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:51,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:51,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-26T16:42:51,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-26T16:42:52,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-26T16:42:52,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-26T16:42:52,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 161 msec 2024-11-26T16:42:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:52,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T16:42:52,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:42:52,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:52,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:42:52,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:52,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:42:52,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:52,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/844b8bcee6a543938a18d23f1713b7cc is 50, key is test_row_0/A:col10/1732639372027/Put/seqid=0 2024-11-26T16:42:52,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742297_1473 (size=14541) 2024-11-26T16:42:52,096 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/17e46581743f484687fd0200712d0069 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/17e46581743f484687fd0200712d0069 2024-11-26T16:42:52,100 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/A of 42f6c759eca28a5c6a511cb6fa26803c into 17e46581743f484687fd0200712d0069(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:52,100 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:52,100 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/A, priority=13, startTime=1732639371658; duration=0sec 2024-11-26T16:42:52,100 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:52,100 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:A 2024-11-26T16:42:52,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732639372085, max timestamp=1732639372104Num uniques 859; 2024-11-26T16:42:52,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:42:52,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732639372085, max timestamp=1732639372104Num uniques 859; 2024-11-26T16:42:52,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:42:52,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732639372085, max timestamp=1732639372104Num uniques 859; 2024-11-26T16:42:52,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:42:52,158 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/5c4e5ce368b748a4aa728da71e82f90b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/5c4e5ce368b748a4aa728da71e82f90b 2024-11-26T16:42:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-26T16:42:52,161 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:42:52,161 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:42:52,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-26T16:42:52,163 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:42:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T16:42:52,163 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/C of 42f6c759eca28a5c6a511cb6fa26803c into 5c4e5ce368b748a4aa728da71e82f90b(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:52,163 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:52,163 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/C, priority=13, startTime=1732639371659; duration=0sec 2024-11-26T16:42:52,164 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:52,164 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:C 2024-11-26T16:42:52,164 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:42:52,164 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:42:52,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639432181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:52,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:52,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639432181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:52,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:52,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:52,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639432182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:52,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:52,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T16:42:52,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639432301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:52,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639432301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:52,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639432301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:52,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:52,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:52,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:52,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:52,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:52,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:52,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/844b8bcee6a543938a18d23f1713b7cc 2024-11-26T16:42:52,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/a86048bbd220425d84347673ff04e0c1 is 50, key is test_row_0/B:col10/1732639372027/Put/seqid=0 2024-11-26T16:42:52,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742298_1474 (size=12151) 2024-11-26T16:42:52,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:52,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:52,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:52,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:52,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:52,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T16:42:52,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639432521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:52,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639432521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:52,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639432521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:52,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:52,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:52,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:52,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:52,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:52,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:52,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:52,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:52,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:52,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:52,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:52,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T16:42:52,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639432841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:52,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639432841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:52,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639432841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:52,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/a86048bbd220425d84347673ff04e0c1 2024-11-26T16:42:52,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/a04394af98e747b2a7bc678a32ac9624 is 50, key is test_row_0/C:col10/1732639372027/Put/seqid=0 2024-11-26T16:42:52,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742299_1475 (size=12151) 2024-11-26T16:42:52,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:52,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:52,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:52,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:52,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:52,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:53,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:53,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:53,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:53,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:53,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:53,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:53,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:53,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:53,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:53,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:53,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:53,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:53,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:53,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:53,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/a04394af98e747b2a7bc678a32ac9624 2024-11-26T16:42:53,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/844b8bcee6a543938a18d23f1713b7cc as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/844b8bcee6a543938a18d23f1713b7cc 2024-11-26T16:42:53,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/844b8bcee6a543938a18d23f1713b7cc, entries=200, sequenceid=182, filesize=14.2 K 2024-11-26T16:42:53,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/a86048bbd220425d84347673ff04e0c1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/a86048bbd220425d84347673ff04e0c1 2024-11-26T16:42:53,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/a86048bbd220425d84347673ff04e0c1, entries=150, sequenceid=182, filesize=11.9 K 2024-11-26T16:42:53,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/a04394af98e747b2a7bc678a32ac9624 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a04394af98e747b2a7bc678a32ac9624 2024-11-26T16:42:53,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a04394af98e747b2a7bc678a32ac9624, entries=150, sequenceid=182, filesize=11.9 K 2024-11-26T16:42:53,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 42f6c759eca28a5c6a511cb6fa26803c in 1258ms, sequenceid=182, compaction requested=true 2024-11-26T16:42:53,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:53,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:42:53,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:53,296 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:53,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:42:53,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:53,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:53,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:53,296 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:53,296 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:53,296 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:53,296 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/B is initiating minor compaction (all files) 2024-11-26T16:42:53,296 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/A is initiating minor compaction (all files) 2024-11-26T16:42:53,297 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/A in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:53,297 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/B in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:53,297 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/c8e971330edf4002ab8a501220c06f21, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/4378acfdc44945fb956538b1b95629aa, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/a86048bbd220425d84347673ff04e0c1] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=36.0 K 2024-11-26T16:42:53,297 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/17e46581743f484687fd0200712d0069, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/7c585aa0d209472484be99b1ca2aaa80, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/844b8bcee6a543938a18d23f1713b7cc] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=38.3 K 2024-11-26T16:42:53,297 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting c8e971330edf4002ab8a501220c06f21, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732639369637 2024-11-26T16:42:53,297 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17e46581743f484687fd0200712d0069, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732639369637 2024-11-26T16:42:53,297 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c585aa0d209472484be99b1ca2aaa80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732639370840 2024-11-26T16:42:53,297 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 4378acfdc44945fb956538b1b95629aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732639370840 2024-11-26T16:42:53,298 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a86048bbd220425d84347673ff04e0c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732639372021 2024-11-26T16:42:53,298 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 844b8bcee6a543938a18d23f1713b7cc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732639372021 2024-11-26T16:42:53,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T16:42:53,306 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#A#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:53,306 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/2ce3ef96333448e2b8d266851e7d78ca is 50, key is test_row_0/A:col10/1732639372027/Put/seqid=0 2024-11-26T16:42:53,308 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#B#compaction#402 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:53,308 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/65035fac72f04a1d92d89bbcc0e1b778 is 50, key is test_row_0/B:col10/1732639372027/Put/seqid=0 2024-11-26T16:42:53,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742300_1476 (size=12629) 2024-11-26T16:42:53,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742301_1477 (size=12629) 2024-11-26T16:42:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:53,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-26T16:42:53,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:42:53,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:53,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:42:53,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:53,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:42:53,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:53,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/33abd540d1164ce58b56d87ee869dbc4 is 50, key is test_row_0/A:col10/1732639373366/Put/seqid=0 2024-11-26T16:42:53,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742302_1478 (size=14541) 2024-11-26T16:42:53,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/33abd540d1164ce58b56d87ee869dbc4 2024-11-26T16:42:53,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/9de3af1d2cad4ee4b815f1809e128f74 is 50, key is test_row_0/B:col10/1732639373366/Put/seqid=0 2024-11-26T16:42:53,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:53,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:53,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:53,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:53,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:53,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:53,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:53,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742303_1479 (size=12151) 2024-11-26T16:42:53,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/9de3af1d2cad4ee4b815f1809e128f74 2024-11-26T16:42:53,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/0a901b0da82e4b37bd41ec5320e1c585 is 50, key is test_row_0/C:col10/1732639373366/Put/seqid=0 2024-11-26T16:42:53,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742304_1480 (size=12151) 2024-11-26T16:42:53,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/0a901b0da82e4b37bd41ec5320e1c585 2024-11-26T16:42:53,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:53,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/33abd540d1164ce58b56d87ee869dbc4 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/33abd540d1164ce58b56d87ee869dbc4 2024-11-26T16:42:53,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639433386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:53,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:53,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639433388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:53,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:53,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:53,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:53,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:53,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639433392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:53,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:53,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:53,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:53,411 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:53,411 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:53,411 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:53,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/33abd540d1164ce58b56d87ee869dbc4, entries=200, sequenceid=221, filesize=14.2 K 2024-11-26T16:42:53,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/9de3af1d2cad4ee4b815f1809e128f74 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/9de3af1d2cad4ee4b815f1809e128f74 2024-11-26T16:42:53,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/9de3af1d2cad4ee4b815f1809e128f74, entries=150, sequenceid=221, filesize=11.9 K 2024-11-26T16:42:53,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/0a901b0da82e4b37bd41ec5320e1c585 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/0a901b0da82e4b37bd41ec5320e1c585 2024-11-26T16:42:53,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/0a901b0da82e4b37bd41ec5320e1c585, entries=150, sequenceid=221, filesize=11.9 K 2024-11-26T16:42:53,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 42f6c759eca28a5c6a511cb6fa26803c in 55ms, sequenceid=221, compaction requested=true 2024-11-26T16:42:53,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:53,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:A, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:53,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-26T16:42:53,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:B, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:53,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-26T16:42:53,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:53,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-26T16:42:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:53,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-26T16:42:53,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:42:53,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:53,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:42:53,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:53,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:42:53,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:53,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/e74285539b3440a39ec242857a633139 is 50, key is test_row_0/A:col10/1732639373386/Put/seqid=0 2024-11-26T16:42:53,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742305_1481 (size=14541) 2024-11-26T16:42:53,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:53,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:53,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:53,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:53,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:53,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:53,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:53,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732639373578, max timestamp=1732639373598Num uniques 859; 2024-11-26T16:42:53,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T16:42:53,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732639373578, max timestamp=1732639373598Num uniques 859; 2024-11-26T16:42:53,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732639373578, max timestamp=1732639373598Num uniques 859; 2024-11-26T16:42:53,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T16:42:53,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T16:42:53,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:53,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639433656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:53,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639433661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:53,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:53,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639433662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:53,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:53,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:53,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:53,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:53,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:53,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:53,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:53,721 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/2ce3ef96333448e2b8d266851e7d78ca as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/2ce3ef96333448e2b8d266851e7d78ca 2024-11-26T16:42:53,723 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/65035fac72f04a1d92d89bbcc0e1b778 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/65035fac72f04a1d92d89bbcc0e1b778 2024-11-26T16:42:53,726 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/A of 42f6c759eca28a5c6a511cb6fa26803c into 2ce3ef96333448e2b8d266851e7d78ca(size=12.3 K), total size for store is 26.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:53,726 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:53,726 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/A, priority=13, startTime=1732639373295; duration=0sec 2024-11-26T16:42:53,726 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-26T16:42:53,726 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:A 2024-11-26T16:42:53,726 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:A 2024-11-26T16:42:53,726 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:53,727 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:53,727 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/C is initiating minor compaction (all files) 2024-11-26T16:42:53,727 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/C in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:53,727 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/5c4e5ce368b748a4aa728da71e82f90b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/117a7871cadd4d04a906aa56659682d5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a04394af98e747b2a7bc678a32ac9624, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/0a901b0da82e4b37bd41ec5320e1c585] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=47.8 K 2024-11-26T16:42:53,728 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c4e5ce368b748a4aa728da71e82f90b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732639369637 2024-11-26T16:42:53,728 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 117a7871cadd4d04a906aa56659682d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732639370840 2024-11-26T16:42:53,728 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting a04394af98e747b2a7bc678a32ac9624, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732639372021 2024-11-26T16:42:53,728 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a901b0da82e4b37bd41ec5320e1c585, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732639372178 2024-11-26T16:42:53,729 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/B of 42f6c759eca28a5c6a511cb6fa26803c into 65035fac72f04a1d92d89bbcc0e1b778(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:53,729 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:53,729 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/B, priority=13, startTime=1732639373296; duration=0sec 2024-11-26T16:42:53,729 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-26T16:42:53,729 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:B 2024-11-26T16:42:53,729 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:B 2024-11-26T16:42:53,729 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-26T16:42:53,730 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T16:42:53,730 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T16:42:53,730 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. because compaction request was cancelled 2024-11-26T16:42:53,730 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:C 2024-11-26T16:42:53,730 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-26T16:42:53,731 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T16:42:53,731 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T16:42:53,731 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. because compaction request was cancelled 2024-11-26T16:42:53,731 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:B 2024-11-26T16:42:53,731 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-26T16:42:53,732 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T16:42:53,732 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T16:42:53,732 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. because compaction request was cancelled 2024-11-26T16:42:53,732 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:A 2024-11-26T16:42:53,736 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#C#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:53,737 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/28755bd78fba4b889b85f1661eab1ad5 is 50, key is test_row_0/C:col10/1732639373366/Put/seqid=0 2024-11-26T16:42:53,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742306_1482 (size=12663) 2024-11-26T16:42:53,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639433781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:53,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:53,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:53,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639433781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:53,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:53,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639433791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:53,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:53,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:53,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:53,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:53,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:53,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:53,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/e74285539b3440a39ec242857a633139 2024-11-26T16:42:53,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/233e3ec1efc1451c8418e09adffea896 is 50, key is test_row_0/B:col10/1732639373386/Put/seqid=0 2024-11-26T16:42:53,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742307_1483 (size=12151) 2024-11-26T16:42:53,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:53,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639433991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:53,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:53,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:53,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:54,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:54,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:54,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:54,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:54,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639434001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:54,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:54,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639434011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:54,147 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/28755bd78fba4b889b85f1661eab1ad5 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/28755bd78fba4b889b85f1661eab1ad5 2024-11-26T16:42:54,152 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/C of 42f6c759eca28a5c6a511cb6fa26803c into 28755bd78fba4b889b85f1661eab1ad5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:54,152 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:54,153 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/C, priority=12, startTime=1732639373422; duration=0sec 2024-11-26T16:42:54,153 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:54,153 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:C 2024-11-26T16:42:54,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:54,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:54,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:54,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:54,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T16:42:54,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:54,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:54,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:54,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639434311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:54,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:54,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639434321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:54,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:54,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639434331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:54,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/233e3ec1efc1451c8418e09adffea896 2024-11-26T16:42:54,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:54,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/aa277131ae0141298bce024ef6fd6fe3 is 50, key is test_row_0/C:col10/1732639373386/Put/seqid=0 2024-11-26T16:42:54,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742308_1484 (size=12151) 2024-11-26T16:42:54,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:54,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:54,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:54,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:54,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44016 deadline: 1732639434461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:54,474 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in 42f6c759eca28a5c6a511cb6fa26803c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:54,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:54,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44050 deadline: 1732639434501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:54,507 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in 42f6c759eca28a5c6a511cb6fa26803c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:42:54,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:54,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:54,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:54,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:54,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:54,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/aa277131ae0141298bce024ef6fd6fe3 2024-11-26T16:42:54,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/e74285539b3440a39ec242857a633139 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/e74285539b3440a39ec242857a633139 2024-11-26T16:42:54,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/e74285539b3440a39ec242857a633139, entries=200, sequenceid=231, filesize=14.2 K 2024-11-26T16:42:54,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/233e3ec1efc1451c8418e09adffea896 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/233e3ec1efc1451c8418e09adffea896 2024-11-26T16:42:54,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/233e3ec1efc1451c8418e09adffea896, entries=150, sequenceid=231, filesize=11.9 K 2024-11-26T16:42:54,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/aa277131ae0141298bce024ef6fd6fe3 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/aa277131ae0141298bce024ef6fd6fe3 2024-11-26T16:42:54,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:54,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:54,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:54,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:54,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/aa277131ae0141298bce024ef6fd6fe3, entries=150, sequenceid=231, filesize=11.9 K 2024-11-26T16:42:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 42f6c759eca28a5c6a511cb6fa26803c in 1255ms, sequenceid=231, compaction requested=true 2024-11-26T16:42:54,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:54,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:42:54,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:54,774 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:54,775 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:54,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:42:54,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:54,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:54,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:54,776 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:54,776 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:54,776 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/A is initiating minor compaction (all files) 2024-11-26T16:42:54,776 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/B is initiating minor compaction (all files) 2024-11-26T16:42:54,776 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/A in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:54,776 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/B in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:54,776 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/2ce3ef96333448e2b8d266851e7d78ca, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/33abd540d1164ce58b56d87ee869dbc4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/e74285539b3440a39ec242857a633139] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=40.7 K 2024-11-26T16:42:54,776 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/65035fac72f04a1d92d89bbcc0e1b778, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/9de3af1d2cad4ee4b815f1809e128f74, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/233e3ec1efc1451c8418e09adffea896] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=36.1 K 2024-11-26T16:42:54,776 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ce3ef96333448e2b8d266851e7d78ca, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732639372021 2024-11-26T16:42:54,777 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 65035fac72f04a1d92d89bbcc0e1b778, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732639372021 2024-11-26T16:42:54,777 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33abd540d1164ce58b56d87ee869dbc4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732639372178 2024-11-26T16:42:54,777 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 9de3af1d2cad4ee4b815f1809e128f74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732639372178 2024-11-26T16:42:54,777 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting e74285539b3440a39ec242857a633139, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732639373369 2024-11-26T16:42:54,778 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 233e3ec1efc1451c8418e09adffea896, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732639373386 2024-11-26T16:42:54,784 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#A#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:54,785 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/af8c6799e1e641429e866c101602226b is 50, key is test_row_0/A:col10/1732639373386/Put/seqid=0 2024-11-26T16:42:54,785 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#B#compaction#412 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:54,786 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/83575546c9fe409e943a11eee0bbf7eb is 50, key is test_row_0/B:col10/1732639373386/Put/seqid=0 2024-11-26T16:42:54,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742310_1486 (size=12731) 2024-11-26T16:42:54,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742309_1485 (size=12731) 2024-11-26T16:42:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:54,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-26T16:42:54,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:42:54,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:54,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:42:54,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:54,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:42:54,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:54,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/9e052426db0048c2a7fdea80dd669d4f is 50, key is test_row_0/A:col10/1732639373640/Put/seqid=0 2024-11-26T16:42:54,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742311_1487 (size=14741) 2024-11-26T16:42:54,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639434857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:54,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639434860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:54,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639434861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:54,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:54,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:54,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:54,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:54,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:54,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:54,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:54,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639434971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:54,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639434981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:54,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639434981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:55,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:55,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:55,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:55,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:55,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:55,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:55,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639435191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:55,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639435191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:55,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639435191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:55,201 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/83575546c9fe409e943a11eee0bbf7eb as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/83575546c9fe409e943a11eee0bbf7eb 2024-11-26T16:42:55,201 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/af8c6799e1e641429e866c101602226b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/af8c6799e1e641429e866c101602226b 2024-11-26T16:42:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:55,206 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/B of 42f6c759eca28a5c6a511cb6fa26803c into 83575546c9fe409e943a11eee0bbf7eb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:55,206 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:55,206 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/B, priority=13, startTime=1732639374774; duration=0sec 2024-11-26T16:42:55,206 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/A of 42f6c759eca28a5c6a511cb6fa26803c into af8c6799e1e641429e866c101602226b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:55,206 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:55,206 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/A, priority=13, startTime=1732639374774; duration=0sec 2024-11-26T16:42:55,206 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:55,206 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:B 2024-11-26T16:42:55,206 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:55,206 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:A 2024-11-26T16:42:55,206 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-26T16:42:55,207 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T16:42:55,207 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T16:42:55,207 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. because compaction request was cancelled 2024-11-26T16:42:55,207 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:C 2024-11-26T16:42:55,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:55,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:55,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:55,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:55,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:55,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:55,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:55,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/9e052426db0048c2a7fdea80dd669d4f 2024-11-26T16:42:55,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/93f625bba304467cbc6a008cc1d36028 is 50, key is test_row_0/B:col10/1732639373640/Put/seqid=0 2024-11-26T16:42:55,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742312_1488 (size=12301) 2024-11-26T16:42:55,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:55,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:55,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:55,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:55,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:55,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:55,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:55,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639435511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:55,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639435511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:55,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:55,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:55,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:55,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:55,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639435511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:55,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:55,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:55,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:55,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:55,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:55,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:55,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:55,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:55,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:55,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:55,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:42:55,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/93f625bba304467cbc6a008cc1d36028 2024-11-26T16:42:55,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/c515110182ce406d94563b8f8de5d093 is 50, key is test_row_0/C:col10/1732639373640/Put/seqid=0 2024-11-26T16:42:55,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742313_1489 (size=12301) 2024-11-26T16:42:55,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/c515110182ce406d94563b8f8de5d093 2024-11-26T16:42:55,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/9e052426db0048c2a7fdea80dd669d4f as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/9e052426db0048c2a7fdea80dd669d4f 2024-11-26T16:42:55,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/9e052426db0048c2a7fdea80dd669d4f, entries=200, sequenceid=270, filesize=14.4 K 2024-11-26T16:42:55,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/93f625bba304467cbc6a008cc1d36028 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/93f625bba304467cbc6a008cc1d36028 2024-11-26T16:42:55,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/93f625bba304467cbc6a008cc1d36028, entries=150, sequenceid=270, filesize=12.0 K 2024-11-26T16:42:55,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/c515110182ce406d94563b8f8de5d093 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/c515110182ce406d94563b8f8de5d093 2024-11-26T16:42:55,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/c515110182ce406d94563b8f8de5d093, entries=150, sequenceid=270, filesize=12.0 K 2024-11-26T16:42:55,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 42f6c759eca28a5c6a511cb6fa26803c in 853ms, sequenceid=270, compaction requested=true 2024-11-26T16:42:55,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:55,682 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-26T16:42:55,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:42:55,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:55,682 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-26T16:42:55,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:42:55,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:55,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:55,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:55,683 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T16:42:55,683 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T16:42:55,683 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. because compaction request was cancelled 2024-11-26T16:42:55,683 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:A 2024-11-26T16:42:55,683 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-26T16:42:55,683 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:55,683 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-26T16:42:55,683 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. because compaction request was cancelled 2024-11-26T16:42:55,683 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:B 2024-11-26T16:42:55,684 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:55,684 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/C is initiating minor compaction (all files) 2024-11-26T16:42:55,684 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/C in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:55,684 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/28755bd78fba4b889b85f1661eab1ad5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/aa277131ae0141298bce024ef6fd6fe3, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/c515110182ce406d94563b8f8de5d093] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=36.2 K 2024-11-26T16:42:55,684 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28755bd78fba4b889b85f1661eab1ad5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732639372178 2024-11-26T16:42:55,685 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa277131ae0141298bce024ef6fd6fe3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732639373386 2024-11-26T16:42:55,685 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting c515110182ce406d94563b8f8de5d093, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732639373640 2024-11-26T16:42:55,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-26T16:42:55,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:55,691 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-26T16:42:55,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:42:55,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:55,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:42:55,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:55,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:42:55,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:42:55,692 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#C#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:55,693 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/1400b9993c2b44bea2e440e57f69a0b5 is 50, key is test_row_0/C:col10/1732639373640/Put/seqid=0 2024-11-26T16:42:55,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/8859c4624a144e34924ffc7dca06c3c6 is 50, key is test_row_0/A:col10/1732639374859/Put/seqid=0 2024-11-26T16:42:55,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742314_1490 (size=12915) 2024-11-26T16:42:55,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742315_1491 (size=12301) 2024-11-26T16:42:55,712 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/8859c4624a144e34924ffc7dca06c3c6 2024-11-26T16:42:55,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/19a9416609a04c58a3668d38d11f1145 is 50, key is test_row_0/B:col10/1732639374859/Put/seqid=0 2024-11-26T16:42:55,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742316_1492 (size=12301) 2024-11-26T16:42:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:56,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:42:56,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639376073, max timestamp=1732639376091Num uniques 859; 2024-11-26T16:42:56,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639376073, max timestamp=1732639376091Num uniques 859; 2024-11-26T16:42:56,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:56,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:42:56,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732639376069, max timestamp=1732639376091Num uniques 859; 2024-11-26T16:42:56,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T16:42:56,116 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/1400b9993c2b44bea2e440e57f69a0b5 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/1400b9993c2b44bea2e440e57f69a0b5 2024-11-26T16:42:56,121 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/C of 42f6c759eca28a5c6a511cb6fa26803c into 1400b9993c2b44bea2e440e57f69a0b5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:56,121 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:56,121 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/C, priority=13, startTime=1732639375682; duration=0sec 2024-11-26T16:42:56,121 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:56,121 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:C 2024-11-26T16:42:56,127 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/19a9416609a04c58a3668d38d11f1145 2024-11-26T16:42:56,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/a0bf18bb789e4ff0a6c6460b8b0b3a01 is 50, key is test_row_0/C:col10/1732639374859/Put/seqid=0 2024-11-26T16:42:56,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742317_1493 (size=12301) 2024-11-26T16:42:56,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639436153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:56,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:56,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639436160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:56,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:56,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:56,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:56,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639436169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:56,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639436271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:56,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639436281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:56,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:56,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:56,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:56,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639436291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T16:42:56,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639436491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:56,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:56,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:56,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:56,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639436501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:56,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:56,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:56,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:56,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639436511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:56,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:56,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:56,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:56,541 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/a0bf18bb789e4ff0a6c6460b8b0b3a01 2024-11-26T16:42:56,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/8859c4624a144e34924ffc7dca06c3c6 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/8859c4624a144e34924ffc7dca06c3c6 2024-11-26T16:42:56,548 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/8859c4624a144e34924ffc7dca06c3c6, entries=150, sequenceid=281, filesize=12.0 K 2024-11-26T16:42:56,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/19a9416609a04c58a3668d38d11f1145 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/19a9416609a04c58a3668d38d11f1145 2024-11-26T16:42:56,552 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/19a9416609a04c58a3668d38d11f1145, entries=150, sequenceid=281, filesize=12.0 K 2024-11-26T16:42:56,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/a0bf18bb789e4ff0a6c6460b8b0b3a01 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a0bf18bb789e4ff0a6c6460b8b0b3a01 2024-11-26T16:42:56,556 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a0bf18bb789e4ff0a6c6460b8b0b3a01, entries=150, sequenceid=281, filesize=12.0 K 2024-11-26T16:42:56,557 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=138.65 KB/141980 for 42f6c759eca28a5c6a511cb6fa26803c in 866ms, sequenceid=281, compaction requested=true 2024-11-26T16:42:56,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:56,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:56,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-26T16:42:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-26T16:42:56,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-26T16:42:56,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.3940 sec 2024-11-26T16:42:56,562 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 4.4000 sec 2024-11-26T16:42:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:56,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=145.36 KB heapSize=415.77 KB 2024-11-26T16:42:56,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732639376152, max timestamp=1732639376820Num uniques 1018; 2024-11-26T16:42:56,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732639376152, max timestamp=1732639376820Num uniques 1118; 2024-11-26T16:42:56,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-26T16:42:56,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732639376152, max timestamp=1732639376820Num uniques 1018; 2024-11-26T16:42:56,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T16:42:56,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-26T16:42:56,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:42:56,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:56,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:42:56,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:56,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:42:56,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:56,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/c74ea5f927eb4ec39a13a8ade3aabd8a is 50, key is test_row_0/A:col10/1732639376820/Put/seqid=0 2024-11-26T16:42:56,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742318_1494 (size=17181) 2024-11-26T16:42:56,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732639376884, max timestamp=1732639376901Num uniques 859; 2024-11-26T16:42:56,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:42:56,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732639376884, max timestamp=1732639376901Num uniques 859; 2024-11-26T16:42:56,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:42:56,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732639376884, max timestamp=1732639376901Num uniques 859; 2024-11-26T16:42:56,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:42:56,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732639376919, max timestamp=1732639376985Num uniques 1068; 2024-11-26T16:42:56,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732639376919, max timestamp=1732639376985Num uniques 1068; 2024-11-26T16:42:56,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-26T16:42:56,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732639376919, max timestamp=1732639376985Num uniques 1068; 2024-11-26T16:42:56,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-26T16:42:56,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-26T16:42:57,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639437070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:57,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732639377057, max timestamp=1732639377075Num uniques 1077; 2024-11-26T16:42:57,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732639377057, max timestamp=1732639377075Num uniques 1077; 2024-11-26T16:42:57,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-26T16:42:57,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-26T16:42:57,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732639377057, max timestamp=1732639377075Num uniques 1077; 2024-11-26T16:42:57,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-26T16:42:57,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732639377161, max timestamp=1732639377201Num uniques 1036; 2024-11-26T16:42:57,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732639377161, max timestamp=1732639377201Num uniques 1036; 2024-11-26T16:42:57,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-26T16:42:57,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-26T16:42:57,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732639377161, max timestamp=1732639377201Num uniques 1036; 2024-11-26T16:42:57,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-26T16:42:57,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/c74ea5f927eb4ec39a13a8ade3aabd8a 2024-11-26T16:42:57,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/d702d21cbea44a85b9d881db726e15df is 50, key is test_row_0/B:col10/1732639376820/Put/seqid=0 2024-11-26T16:42:57,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742319_1495 (size=12301) 2024-11-26T16:42:57,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/d702d21cbea44a85b9d881db726e15df 2024-11-26T16:42:57,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/b0da9c9a7a354463883a05629a900bae is 50, key is test_row_0/C:col10/1732639376820/Put/seqid=0 2024-11-26T16:42:57,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742320_1496 (size=12301) 2024-11-26T16:42:57,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732639377248, max timestamp=1732639377280Num uniques 1095; 2024-11-26T16:42:57,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732639377248, max timestamp=1732639377280Num uniques 1095; 2024-11-26T16:42:57,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-26T16:42:57,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-26T16:42:57,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732639377248, max timestamp=1732639377280Num uniques 1095; 2024-11-26T16:42:57,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-26T16:42:57,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639437352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:57,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639437353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:57,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:57,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:57,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:57,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639437354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:57,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639437461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:57,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:57,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:57,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:57,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639437470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:57,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:57,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:57,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:57,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639437481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:57,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/b0da9c9a7a354463883a05629a900bae 2024-11-26T16:42:57,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/c74ea5f927eb4ec39a13a8ade3aabd8a as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c74ea5f927eb4ec39a13a8ade3aabd8a 2024-11-26T16:42:57,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c74ea5f927eb4ec39a13a8ade3aabd8a, entries=250, sequenceid=321, filesize=16.8 K 2024-11-26T16:42:57,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/d702d21cbea44a85b9d881db726e15df as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/d702d21cbea44a85b9d881db726e15df 2024-11-26T16:42:57,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/d702d21cbea44a85b9d881db726e15df, entries=150, sequenceid=321, filesize=12.0 K 2024-11-26T16:42:57,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/b0da9c9a7a354463883a05629a900bae as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/b0da9c9a7a354463883a05629a900bae 2024-11-26T16:42:57,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/b0da9c9a7a354463883a05629a900bae, entries=150, sequenceid=321, filesize=12.0 K 2024-11-26T16:42:57,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=154.31 KB/158010 for 42f6c759eca28a5c6a511cb6fa26803c in 854ms, sequenceid=321, compaction requested=true 2024-11-26T16:42:57,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:57,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:42:57,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:57,676 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:57,676 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:42:57,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:42:57,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:57,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:57,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:57,677 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:57,677 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:42:57,677 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/B is initiating minor compaction (all files) 2024-11-26T16:42:57,677 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/A is initiating minor compaction (all files) 2024-11-26T16:42:57,677 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/A in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:57,677 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/B in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:57,677 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/af8c6799e1e641429e866c101602226b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/9e052426db0048c2a7fdea80dd669d4f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/8859c4624a144e34924ffc7dca06c3c6, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c74ea5f927eb4ec39a13a8ade3aabd8a] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=55.6 K 2024-11-26T16:42:57,677 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/83575546c9fe409e943a11eee0bbf7eb, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/93f625bba304467cbc6a008cc1d36028, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/19a9416609a04c58a3668d38d11f1145, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/d702d21cbea44a85b9d881db726e15df] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=48.5 K 2024-11-26T16:42:57,677 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting af8c6799e1e641429e866c101602226b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732639373386 2024-11-26T16:42:57,677 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 83575546c9fe409e943a11eee0bbf7eb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732639373386 2024-11-26T16:42:57,678 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e052426db0048c2a7fdea80dd669d4f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732639373633 2024-11-26T16:42:57,678 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 93f625bba304467cbc6a008cc1d36028, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732639373640 2024-11-26T16:42:57,678 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8859c4624a144e34924ffc7dca06c3c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732639374855 2024-11-26T16:42:57,678 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 19a9416609a04c58a3668d38d11f1145, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732639374855 2024-11-26T16:42:57,678 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting d702d21cbea44a85b9d881db726e15df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732639376167 2024-11-26T16:42:57,678 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting c74ea5f927eb4ec39a13a8ade3aabd8a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732639376152 2024-11-26T16:42:57,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:57,690 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#A#compaction#423 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:57,690 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/8be02f63ce764bd7af2d17545aadd27e is 50, key is test_row_0/A:col10/1732639376820/Put/seqid=0 2024-11-26T16:42:57,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=161.02 KB heapSize=452.63 KB 2024-11-26T16:42:57,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732639377340, max timestamp=1732639377686Num uniques 1154; 2024-11-26T16:42:57,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-26T16:42:57,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732639377340, max timestamp=1732639377686Num uniques 1154; 2024-11-26T16:42:57,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:42:57,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-26T16:42:57,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732639377340, max timestamp=1732639377686Num uniques 1154; 2024-11-26T16:42:57,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-26T16:42:57,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:57,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:42:57,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:57,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:42:57,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:57,695 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#B#compaction#424 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:57,696 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/0da7c5cf5bf247988845b59439a086cf is 50, key is test_row_0/B:col10/1732639376820/Put/seqid=0 2024-11-26T16:42:57,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/a6b606eb65e84c1ea054e558eb433190 is 50, key is test_row_0/A:col10/1732639377351/Put/seqid=0 2024-11-26T16:42:57,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742321_1497 (size=13017) 2024-11-26T16:42:57,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742323_1499 (size=14741) 2024-11-26T16:42:57,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/a6b606eb65e84c1ea054e558eb433190 2024-11-26T16:42:57,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742322_1498 (size=13017) 2024-11-26T16:42:57,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/6518704ad4164404b11da7393ef5db06 is 50, key is test_row_0/B:col10/1732639377351/Put/seqid=0 2024-11-26T16:42:57,719 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/0da7c5cf5bf247988845b59439a086cf as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/0da7c5cf5bf247988845b59439a086cf 2024-11-26T16:42:57,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742324_1500 (size=12301) 2024-11-26T16:42:57,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/6518704ad4164404b11da7393ef5db06 2024-11-26T16:42:57,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/ea92a6753cf94506b340ba6981759ffe is 50, key is test_row_0/C:col10/1732639377351/Put/seqid=0 2024-11-26T16:42:57,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742325_1501 (size=12301) 2024-11-26T16:42:57,742 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/B of 42f6c759eca28a5c6a511cb6fa26803c into 0da7c5cf5bf247988845b59439a086cf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:57,743 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:57,743 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/B, priority=12, startTime=1732639377675; duration=0sec 2024-11-26T16:42:57,743 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:57,743 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:B 2024-11-26T16:42:57,743 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:57,744 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:57,744 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/C is initiating minor compaction (all files) 2024-11-26T16:42:57,744 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/C in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:57,744 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/1400b9993c2b44bea2e440e57f69a0b5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a0bf18bb789e4ff0a6c6460b8b0b3a01, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/b0da9c9a7a354463883a05629a900bae] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=36.6 K 2024-11-26T16:42:57,745 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 1400b9993c2b44bea2e440e57f69a0b5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732639373640 2024-11-26T16:42:57,745 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a0bf18bb789e4ff0a6c6460b8b0b3a01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732639374855 2024-11-26T16:42:57,745 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting b0da9c9a7a354463883a05629a900bae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732639376167 2024-11-26T16:42:57,752 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#C#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:57,753 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/d8214eb5d3154112bddfb3db3e5cfa90 is 50, key is test_row_0/C:col10/1732639376820/Put/seqid=0 2024-11-26T16:42:57,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742326_1502 (size=13017) 2024-11-26T16:42:57,773 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/d8214eb5d3154112bddfb3db3e5cfa90 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/d8214eb5d3154112bddfb3db3e5cfa90 2024-11-26T16:42:57,779 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/C of 42f6c759eca28a5c6a511cb6fa26803c into d8214eb5d3154112bddfb3db3e5cfa90(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:57,779 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:57,779 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/C, priority=13, startTime=1732639377676; duration=0sec 2024-11-26T16:42:57,779 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:57,779 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:C 2024-11-26T16:42:57,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732639377761, max timestamp=1732639377795Num uniques 859; 2024-11-26T16:42:57,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732639377761, max timestamp=1732639377795Num uniques 859; 2024-11-26T16:42:57,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:42:57,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:42:57,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732639377761, max timestamp=1732639377795Num uniques 859; 2024-11-26T16:42:57,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:42:57,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732639377831, max timestamp=1732639377875Num uniques 1068; 2024-11-26T16:42:57,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-26T16:42:57,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732639377831, max timestamp=1732639377875Num uniques 1068; 2024-11-26T16:42:57,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-26T16:42:57,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732639377831, max timestamp=1732639377875Num uniques 1068; 2024-11-26T16:42:57,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-26T16:42:57,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:57,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639437946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:57,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:57,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:57,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:57,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732639377927, max timestamp=1732639377957Num uniques 1177; 2024-11-26T16:42:57,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-11-26T16:42:57,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732639377927, max timestamp=1732639377957Num uniques 1177; 2024-11-26T16:42:57,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-11-26T16:42:57,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732639377927, max timestamp=1732639377957Num uniques 1177; 2024-11-26T16:42:57,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-11-26T16:42:58,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732639378034, max timestamp=1732639378065Num uniques 1136; 2024-11-26T16:42:58,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-26T16:42:58,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732639378034, max timestamp=1732639378065Num uniques 1136; 2024-11-26T16:42:58,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-26T16:42:58,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732639378045, max timestamp=1732639378065Num uniques 1136; 2024-11-26T16:42:58,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-11-26T16:42:58,109 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/8be02f63ce764bd7af2d17545aadd27e as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/8be02f63ce764bd7af2d17545aadd27e 2024-11-26T16:42:58,113 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/A of 42f6c759eca28a5c6a511cb6fa26803c into 8be02f63ce764bd7af2d17545aadd27e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:58,113 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:58,113 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/A, priority=12, startTime=1732639377675; duration=0sec 2024-11-26T16:42:58,113 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:58,113 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:A 2024-11-26T16:42:58,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/ea92a6753cf94506b340ba6981759ffe 2024-11-26T16:42:58,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/a6b606eb65e84c1ea054e558eb433190 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a6b606eb65e84c1ea054e558eb433190 2024-11-26T16:42:58,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a6b606eb65e84c1ea054e558eb433190, entries=200, sequenceid=428, filesize=14.4 K 2024-11-26T16:42:58,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/6518704ad4164404b11da7393ef5db06 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/6518704ad4164404b11da7393ef5db06 2024-11-26T16:42:58,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/6518704ad4164404b11da7393ef5db06, entries=150, sequenceid=428, filesize=12.0 K 2024-11-26T16:42:58,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/ea92a6753cf94506b340ba6981759ffe as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/ea92a6753cf94506b340ba6981759ffe 2024-11-26T16:42:58,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/ea92a6753cf94506b340ba6981759ffe, entries=150, sequenceid=428, filesize=12.0 K 2024-11-26T16:42:58,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.66 KB/68256, currentSize=138.65 KB/141980 for 42f6c759eca28a5c6a511cb6fa26803c in 465ms, sequenceid=428, compaction requested=false 2024-11-26T16:42:58,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:58,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:58,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732639378119, max timestamp=1732639378139Num uniques 1095; 2024-11-26T16:42:58,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-26T16:42:58,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732639378119, max timestamp=1732639378139Num uniques 1095; 2024-11-26T16:42:58,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-26T16:42:58,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732639378119, max timestamp=1732639378139Num uniques 1045; 2024-11-26T16:42:58,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-26T16:42:58,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=26.84 KB heapSize=62.86 KB 2024-11-26T16:42:58,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:42:58,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:58,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:42:58,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:58,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:42:58,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:58,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/c9ad89990db244468bd77fd76c7835ce is 50, key is test_row_0/A:col10/1732639378161/Put/seqid=0 2024-11-26T16:42:58,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742327_1503 (size=19621) 2024-11-26T16:42:58,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732639378200, max timestamp=1732639378231Num uniques 859; 2024-11-26T16:42:58,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T16:42:58,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732639378200, max timestamp=1732639378231Num uniques 859; 2024-11-26T16:42:58,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T16:42:58,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732639378200, max timestamp=1732639378231Num uniques 859; 2024-11-26T16:42:58,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-26T16:42:58,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9968, totalHeapSize=22912, min timestamp=1732639378287, max timestamp=1732639378336Num uniques 1118; 2024-11-26T16:42:58,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51188, new segment data size=9968, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-26T16:42:58,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9969, totalHeapSize=22912, min timestamp=1732639378287, max timestamp=1732639378336Num uniques 1118; 2024-11-26T16:42:58,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51189, new segment data size=9969, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-26T16:42:58,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732639378287, max timestamp=1732639378336Num uniques 1118; 2024-11-26T16:42:58,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51188, new segment data size=6870, suffix heap size=144240,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=150 2024-11-26T16:42:58,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639438434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:58,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:58,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732639378416, max timestamp=1732639378441Num uniques 1077; 2024-11-26T16:42:58,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-26T16:42:58,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732639378416, max timestamp=1732639378441Num uniques 1077; 2024-11-26T16:42:58,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-26T16:42:58,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732639378416, max timestamp=1732639378441Num uniques 1009; 2024-11-26T16:42:58,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46224, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-11-26T16:42:58,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732639378452, max timestamp=1732639378539Num uniques 1086; 2024-11-26T16:42:58,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-26T16:42:58,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732639378452, max timestamp=1732639378539Num uniques 1086; 2024-11-26T16:42:58,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-26T16:42:58,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732639378452, max timestamp=1732639378539Num uniques 1086; 2024-11-26T16:42:58,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-26T16:42:58,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/c9ad89990db244468bd77fd76c7835ce 2024-11-26T16:42:58,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/7b7768b80f4543859e064a32ec10a5f6 is 50, key is test_row_0/B:col10/1732639378161/Put/seqid=0 2024-11-26T16:42:58,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742328_1504 (size=12301) 2024-11-26T16:42:58,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732639378595, max timestamp=1732639378628Num uniques 1095; 2024-11-26T16:42:58,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732639378595, max timestamp=1732639378628Num uniques 1095; 2024-11-26T16:42:58,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-26T16:42:58,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-26T16:42:58,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732639378595, max timestamp=1732639378628Num uniques 1095; 2024-11-26T16:42:58,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-26T16:42:58,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639438702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:58,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639438702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:58,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639438718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:58,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:58,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:58,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:58,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639438821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:58,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639438821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:58,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639438840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:58,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:58,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/7b7768b80f4543859e064a32ec10a5f6 2024-11-26T16:42:59,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/5c82f222f14145cf87799e64f3730dd0 is 50, key is test_row_0/C:col10/1732639378161/Put/seqid=0 2024-11-26T16:42:59,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742329_1505 (size=12301) 2024-11-26T16:42:59,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639439041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:59,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639439041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:59,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639439061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:59,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:59,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:59,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:59,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639439351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:59,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:59,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639439351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:59,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:59,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:59,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:59,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:59,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:59,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639439381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:59,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/5c82f222f14145cf87799e64f3730dd0 2024-11-26T16:42:59,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/c9ad89990db244468bd77fd76c7835ce as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c9ad89990db244468bd77fd76c7835ce 2024-11-26T16:42:59,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c9ad89990db244468bd77fd76c7835ce, entries=300, sequenceid=522, filesize=19.2 K 2024-11-26T16:42:59,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/7b7768b80f4543859e064a32ec10a5f6 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/7b7768b80f4543859e064a32ec10a5f6 2024-11-26T16:42:59,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/7b7768b80f4543859e064a32ec10a5f6, entries=150, sequenceid=522, filesize=12.0 K 2024-11-26T16:42:59,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/5c82f222f14145cf87799e64f3730dd0 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/5c82f222f14145cf87799e64f3730dd0 2024-11-26T16:42:59,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/5c82f222f14145cf87799e64f3730dd0, entries=150, sequenceid=522, filesize=12.0 K 2024-11-26T16:42:59,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~102.66 KB/105120, currentSize=147.60 KB/151140 for 42f6c759eca28a5c6a511cb6fa26803c in 1265ms, sequenceid=522, compaction requested=true 2024-11-26T16:42:59,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:59,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:42:59,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:59,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:42:59,426 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:59,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:59,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:42:59,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:59,426 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:59,428 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:59,428 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:59,428 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/A is initiating minor compaction (all files) 2024-11-26T16:42:59,428 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/B is initiating minor compaction (all files) 2024-11-26T16:42:59,428 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/B in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:59,428 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/A in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:59,428 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/0da7c5cf5bf247988845b59439a086cf, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/6518704ad4164404b11da7393ef5db06, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/7b7768b80f4543859e064a32ec10a5f6] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=36.7 K 2024-11-26T16:42:59,428 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/8be02f63ce764bd7af2d17545aadd27e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a6b606eb65e84c1ea054e558eb433190, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c9ad89990db244468bd77fd76c7835ce] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=46.3 K 2024-11-26T16:42:59,429 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 0da7c5cf5bf247988845b59439a086cf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732639376167 2024-11-26T16:42:59,429 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8be02f63ce764bd7af2d17545aadd27e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732639376167 2024-11-26T16:42:59,429 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 6518704ad4164404b11da7393ef5db06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732639377351 2024-11-26T16:42:59,429 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6b606eb65e84c1ea054e558eb433190, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732639377340 2024-11-26T16:42:59,429 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b7768b80f4543859e064a32ec10a5f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1732639378137 2024-11-26T16:42:59,429 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9ad89990db244468bd77fd76c7835ce, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1732639378119 2024-11-26T16:42:59,435 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#B#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:59,435 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#A#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:59,435 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/889fd3be3e5e4d51b8237eede32cfe67 is 50, key is test_row_0/B:col10/1732639378161/Put/seqid=0 2024-11-26T16:42:59,435 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/04a7971484f34cd7999618b036fa5d64 is 50, key is test_row_0/A:col10/1732639378161/Put/seqid=0 2024-11-26T16:42:59,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742330_1506 (size=13119) 2024-11-26T16:42:59,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742331_1507 (size=13119) 2024-11-26T16:42:59,847 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/889fd3be3e5e4d51b8237eede32cfe67 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/889fd3be3e5e4d51b8237eede32cfe67 2024-11-26T16:42:59,849 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/04a7971484f34cd7999618b036fa5d64 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/04a7971484f34cd7999618b036fa5d64 2024-11-26T16:42:59,852 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/B of 42f6c759eca28a5c6a511cb6fa26803c into 889fd3be3e5e4d51b8237eede32cfe67(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:59,852 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:59,852 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/B, priority=13, startTime=1732639379426; duration=0sec 2024-11-26T16:42:59,852 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:42:59,852 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:B 2024-11-26T16:42:59,852 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:42:59,853 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/A of 42f6c759eca28a5c6a511cb6fa26803c into 04a7971484f34cd7999618b036fa5d64(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:42:59,853 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:42:59,853 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/A, priority=13, startTime=1732639379426; duration=0sec 2024-11-26T16:42:59,853 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:42:59,853 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:42:59,853 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:A 2024-11-26T16:42:59,853 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/C is initiating minor compaction (all files) 2024-11-26T16:42:59,853 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/C in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:42:59,854 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/d8214eb5d3154112bddfb3db3e5cfa90, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/ea92a6753cf94506b340ba6981759ffe, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/5c82f222f14145cf87799e64f3730dd0] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=36.7 K 2024-11-26T16:42:59,854 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting d8214eb5d3154112bddfb3db3e5cfa90, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1732639376167 2024-11-26T16:42:59,854 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting ea92a6753cf94506b340ba6981759ffe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1732639377351 2024-11-26T16:42:59,854 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c82f222f14145cf87799e64f3730dd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1732639378137 2024-11-26T16:42:59,860 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#C#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:42:59,861 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/3ec95ae6fab44764846450ff6e2eb699 is 50, key is test_row_0/C:col10/1732639378161/Put/seqid=0 2024-11-26T16:42:59,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742332_1508 (size=13119) 2024-11-26T16:42:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:42:59,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=154.31 KB heapSize=432 KB 2024-11-26T16:42:59,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:42:59,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:59,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:42:59,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:59,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:42:59,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:42:59,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/c508f365e8694c28801ecbf418c0a4c3 is 50, key is test_row_0/A:col10/1732639378716/Put/seqid=0 2024-11-26T16:42:59,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742333_1509 (size=14741) 2024-11-26T16:42:59,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639439897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:59,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639439898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:59,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:42:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639439901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:42:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:42:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:42:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:43:00,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639440011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:43:00,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639440011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:00,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:00,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:00,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:43:00,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639440221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:43:00,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:00,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639440231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:00,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:00,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:00,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:43:00,269 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/3ec95ae6fab44764846450ff6e2eb699 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/3ec95ae6fab44764846450ff6e2eb699 2024-11-26T16:43:00,274 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/C of 42f6c759eca28a5c6a511cb6fa26803c into 3ec95ae6fab44764846450ff6e2eb699(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:43:00,274 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:43:00,274 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/C, priority=13, startTime=1732639379426; duration=0sec 2024-11-26T16:43:00,274 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:00,274 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:C 2024-11-26T16:43:00,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=630 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/c508f365e8694c28801ecbf418c0a4c3 2024-11-26T16:43:00,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/2d6dd1cd400543e2b3ed229497d9572d is 50, key is test_row_0/B:col10/1732639378716/Put/seqid=0 2024-11-26T16:43:00,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742334_1510 (size=12301) 2024-11-26T16:43:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-26T16:43:00,351 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:43:00,352 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:43:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-26T16:43:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T16:43:00,354 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:43:00,354 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:43:00,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:43:00,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T16:43:00,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T16:43:00,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:43:00,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:43:00,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:43:00,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:00,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:00,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639440531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:43:00,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639440551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:00,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:43:00,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T16:43:00,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:43:00,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:43:00,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:43:00,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:00,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T16:43:00,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=630 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/2d6dd1cd400543e2b3ed229497d9572d 2024-11-26T16:43:00,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/51d6c0c457634aa2ab0cca6d4cf6f17b is 50, key is test_row_0/C:col10/1732639378716/Put/seqid=0 2024-11-26T16:43:00,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742335_1511 (size=12301) 2024-11-26T16:43:00,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T16:43:00,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:43:00,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:43:00,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:43:00,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:00,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:00,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44064 deadline: 1732639440911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:00,919 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_0' in 42f6c759eca28a5c6a511cb6fa26803c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:43:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:43:00,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T16:43:00,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:43:00,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:43:00,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:43:00,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:00,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T16:43:01,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43990 deadline: 1732639441041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:01,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:01,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:01,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:43:01,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44002 deadline: 1732639441071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:01,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 , the old value is region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:01,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:01,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., hostname=2dedfe7466e8,38109,1732639238134, seqNum=2 because the exception is null or not the one we care about 2024-11-26T16:43:01,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=630 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/51d6c0c457634aa2ab0cca6d4cf6f17b 2024-11-26T16:43:01,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/c508f365e8694c28801ecbf418c0a4c3 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c508f365e8694c28801ecbf418c0a4c3 2024-11-26T16:43:01,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c508f365e8694c28801ecbf418c0a4c3, entries=200, sequenceid=630, filesize=14.4 K 2024-11-26T16:43:01,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T16:43:01,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:43:01,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:43:01,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:43:01,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:01,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:01,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/2d6dd1cd400543e2b3ed229497d9572d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/2d6dd1cd400543e2b3ed229497d9572d 2024-11-26T16:43:01,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/2d6dd1cd400543e2b3ed229497d9572d, entries=150, sequenceid=630, filesize=12.0 K 2024-11-26T16:43:01,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/51d6c0c457634aa2ab0cca6d4cf6f17b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/51d6c0c457634aa2ab0cca6d4cf6f17b 2024-11-26T16:43:01,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/51d6c0c457634aa2ab0cca6d4cf6f17b, entries=150, sequenceid=630, filesize=12.0 K 2024-11-26T16:43:01,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~431.95 KB/442320, currentSize=26.84 KB/27480 for 42f6c759eca28a5c6a511cb6fa26803c in 1253ms, sequenceid=630, compaction requested=false 2024-11-26T16:43:01,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:43:01,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-26T16:43:01,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:43:01,273 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-26T16:43:01,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:43:01,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:01,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:43:01,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:01,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:43:01,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:01,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/8c7ccfe2f6dc412f9f7889773b76d1b1 is 50, key is test_row_1/A:col10/1732639379897/Put/seqid=0 2024-11-26T16:43:01,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742336_1512 (size=9857) 2024-11-26T16:43:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T16:43:01,690 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=638 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/8c7ccfe2f6dc412f9f7889773b76d1b1 2024-11-26T16:43:01,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/80ed40a3753d4c42bd25bc23c3d14eff is 50, key is test_row_1/B:col10/1732639379897/Put/seqid=0 2024-11-26T16:43:01,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742337_1513 (size=9857) 2024-11-26T16:43:01,718 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=638 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/80ed40a3753d4c42bd25bc23c3d14eff 2024-11-26T16:43:01,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/65be6f88b5164b50aa89588634969405 is 50, key is test_row_1/C:col10/1732639379897/Put/seqid=0 2024-11-26T16:43:01,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742338_1514 (size=9857) 2024-11-26T16:43:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:43:02,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. as already flushing 2024-11-26T16:43:02,113 INFO [Thread-1949 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1949. 2024-11-26T16:43:02,113 DEBUG [Thread-1949 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:43:02,113 DEBUG [Thread-1949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:02,113 DEBUG [Thread-1949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:02,113 INFO [Thread-1968 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1968. 2024-11-26T16:43:02,113 DEBUG [Thread-1968 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:43:02,113 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:02,113 DEBUG [Thread-1968 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:02,114 DEBUG [Thread-1968 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:02,114 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:02,115 INFO [Thread-1962 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1962. 2024-11-26T16:43:02,115 DEBUG [Thread-1962 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:43:02,115 DEBUG [Thread-1962 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:02,115 DEBUG [Thread-1962 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:02,115 INFO [Thread-1955 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1955. 2024-11-26T16:43:02,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:02,115 DEBUG [Thread-1955 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:43:02,115 DEBUG [Thread-1955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:02,115 DEBUG [Thread-1955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:02,116 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:02,116 INFO [Thread-1964 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1964. 2024-11-26T16:43:02,116 DEBUG [Thread-1964 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:43:02,116 DEBUG [Thread-1964 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:02,116 DEBUG [Thread-1964 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:02,116 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:02,123 INFO [Thread-1966 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1966. 2024-11-26T16:43:02,123 DEBUG [Thread-1966 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:43:02,123 DEBUG [Thread-1966 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:02,123 DEBUG [Thread-1966 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:02,123 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:02,124 INFO [Thread-1960 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1960. 2024-11-26T16:43:02,124 DEBUG [Thread-1960 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:43:02,124 DEBUG [Thread-1960 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:02,124 DEBUG [Thread-1960 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:02,124 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:02,138 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=638 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/65be6f88b5164b50aa89588634969405 2024-11-26T16:43:02,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/8c7ccfe2f6dc412f9f7889773b76d1b1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/8c7ccfe2f6dc412f9f7889773b76d1b1 2024-11-26T16:43:02,145 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/8c7ccfe2f6dc412f9f7889773b76d1b1, entries=100, sequenceid=638, filesize=9.6 K 2024-11-26T16:43:02,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/80ed40a3753d4c42bd25bc23c3d14eff as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/80ed40a3753d4c42bd25bc23c3d14eff 2024-11-26T16:43:02,148 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/80ed40a3753d4c42bd25bc23c3d14eff, entries=100, sequenceid=638, filesize=9.6 K 2024-11-26T16:43:02,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/65be6f88b5164b50aa89588634969405 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/65be6f88b5164b50aa89588634969405 2024-11-26T16:43:02,152 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/65be6f88b5164b50aa89588634969405, entries=100, sequenceid=638, filesize=9.6 K 2024-11-26T16:43:02,153 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=73.80 KB/75570 for 42f6c759eca28a5c6a511cb6fa26803c in 880ms, sequenceid=638, compaction requested=true 2024-11-26T16:43:02,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:43:02,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:43:02,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-26T16:43:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-26T16:43:02,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-26T16:43:02,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8000 sec 2024-11-26T16:43:02,157 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.8040 sec 2024-11-26T16:43:02,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-26T16:43:02,501 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:43:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:43:02,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-26T16:43:02,932 INFO [Thread-1951 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1951. 2024-11-26T16:43:02,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:43:02,932 DEBUG [Thread-1951 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:43:02,932 DEBUG [Thread-1951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:02,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:02,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:43:02,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:02,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:43:02,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:02,933 DEBUG [Thread-1951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:02,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:02,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/fbed778084084a3aa0140d96e27400ef is 50, key is test_row_0/A:col10/1732639382931/Put/seqid=0 2024-11-26T16:43:02,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742339_1515 (size=12301) 2024-11-26T16:43:03,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=653 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/fbed778084084a3aa0140d96e27400ef 2024-11-26T16:43:03,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/b9283db3e45b4ae18d74771b65f0e0dd is 50, key is test_row_0/B:col10/1732639382931/Put/seqid=0 2024-11-26T16:43:03,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742340_1516 (size=12301) 2024-11-26T16:43:03,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=653 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/b9283db3e45b4ae18d74771b65f0e0dd 2024-11-26T16:43:03,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/a4d343cf1c1c422a974cd83c617a6bef is 50, key is test_row_0/C:col10/1732639382931/Put/seqid=0 2024-11-26T16:43:03,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742341_1517 (size=12301) 2024-11-26T16:43:04,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=653 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/a4d343cf1c1c422a974cd83c617a6bef 2024-11-26T16:43:04,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/fbed778084084a3aa0140d96e27400ef as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/fbed778084084a3aa0140d96e27400ef 2024-11-26T16:43:04,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/fbed778084084a3aa0140d96e27400ef, entries=150, sequenceid=653, filesize=12.0 K 2024-11-26T16:43:04,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/b9283db3e45b4ae18d74771b65f0e0dd as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/b9283db3e45b4ae18d74771b65f0e0dd 2024-11-26T16:43:04,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/b9283db3e45b4ae18d74771b65f0e0dd, entries=150, sequenceid=653, filesize=12.0 K 2024-11-26T16:43:04,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/a4d343cf1c1c422a974cd83c617a6bef as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a4d343cf1c1c422a974cd83c617a6bef 2024-11-26T16:43:04,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a4d343cf1c1c422a974cd83c617a6bef, entries=150, sequenceid=653, filesize=12.0 K 2024-11-26T16:43:04,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 42f6c759eca28a5c6a511cb6fa26803c in 1243ms, sequenceid=653, compaction requested=true 2024-11-26T16:43:04,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:43:04,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:43:04,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:04,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:43:04,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:04,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42f6c759eca28a5c6a511cb6fa26803c:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:43:04,175 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:43:04,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:43:04,175 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:43:04,176 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47578 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:43:04,176 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50018 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:43:04,176 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/A is initiating minor compaction (all files) 2024-11-26T16:43:04,176 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/B is initiating minor compaction (all files) 2024-11-26T16:43:04,177 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/A in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:43:04,177 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/B in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:43:04,177 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/889fd3be3e5e4d51b8237eede32cfe67, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/2d6dd1cd400543e2b3ed229497d9572d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/80ed40a3753d4c42bd25bc23c3d14eff, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/b9283db3e45b4ae18d74771b65f0e0dd] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=46.5 K 2024-11-26T16:43:04,177 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/04a7971484f34cd7999618b036fa5d64, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c508f365e8694c28801ecbf418c0a4c3, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/8c7ccfe2f6dc412f9f7889773b76d1b1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/fbed778084084a3aa0140d96e27400ef] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=48.8 K 2024-11-26T16:43:04,177 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04a7971484f34cd7999618b036fa5d64, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1732639378137 2024-11-26T16:43:04,177 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 889fd3be3e5e4d51b8237eede32cfe67, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1732639378137 2024-11-26T16:43:04,177 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d6dd1cd400543e2b3ed229497d9572d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=630, earliestPutTs=1732639378698 2024-11-26T16:43:04,177 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting c508f365e8694c28801ecbf418c0a4c3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=630, earliestPutTs=1732639378698 2024-11-26T16:43:04,178 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 80ed40a3753d4c42bd25bc23c3d14eff, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=638, earliestPutTs=1732639379894 2024-11-26T16:43:04,178 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c7ccfe2f6dc412f9f7889773b76d1b1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=638, earliestPutTs=1732639379894 2024-11-26T16:43:04,178 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting b9283db3e45b4ae18d74771b65f0e0dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=653, earliestPutTs=1732639382100 2024-11-26T16:43:04,178 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbed778084084a3aa0140d96e27400ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=653, earliestPutTs=1732639382100 2024-11-26T16:43:04,185 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#A#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:43:04,185 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/8f5728c3f59d4b18a2f56b92fcf75ba3 is 50, key is test_row_0/A:col10/1732639382931/Put/seqid=0 2024-11-26T16:43:04,186 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#B#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:43:04,187 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/d6c9578eb3b44efc8c07be6ce20e63e5 is 50, key is test_row_0/B:col10/1732639382931/Put/seqid=0 2024-11-26T16:43:04,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742342_1518 (size=13255) 2024-11-26T16:43:04,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742343_1519 (size=13255) 2024-11-26T16:43:04,482 INFO [Thread-1957 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1957. 2024-11-26T16:43:04,482 DEBUG [Thread-1957 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:43:04,482 DEBUG [Thread-1957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:04,482 DEBUG [Thread-1957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:04,482 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:04,523 INFO [Thread-1953 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1953. 2024-11-26T16:43:04,523 DEBUG [Thread-1953 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:43:04,523 DEBUG [Thread-1953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:04,523 DEBUG [Thread-1953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:04,523 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:04,523 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T16:43:04,523 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 172 2024-11-26T16:43:04,523 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 158 2024-11-26T16:43:04,523 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-26T16:43:04,523 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 195 2024-11-26T16:43:04,523 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-26T16:43:04,523 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T16:43:04,523 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T16:43:04,523 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1032 2024-11-26T16:43:04,523 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3096 rows 2024-11-26T16:43:04,523 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1031 2024-11-26T16:43:04,523 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3090 rows 2024-11-26T16:43:04,523 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1028 2024-11-26T16:43:04,523 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3082 rows 2024-11-26T16:43:04,523 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1031 2024-11-26T16:43:04,523 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3092 rows 2024-11-26T16:43:04,523 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1032 2024-11-26T16:43:04,523 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3095 rows 2024-11-26T16:43:04,524 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T16:43:04,524 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T16:43:04,524 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:04,524 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:04,524 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:04,525 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T16:43:04,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T16:43:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T16:43:04,527 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639384527"}]},"ts":"1732639384527"} 2024-11-26T16:43:04,528 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T16:43:04,528 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T16:43:04,529 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T16:43:04,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42f6c759eca28a5c6a511cb6fa26803c, UNASSIGN}] 2024-11-26T16:43:04,531 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42f6c759eca28a5c6a511cb6fa26803c, UNASSIGN 2024-11-26T16:43:04,531 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=42f6c759eca28a5c6a511cb6fa26803c, regionState=CLOSING, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:04,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42f6c759eca28a5c6a511cb6fa26803c, UNASSIGN because future has completed 2024-11-26T16:43:04,533 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T16:43:04,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134}] 2024-11-26T16:43:04,593 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/8f5728c3f59d4b18a2f56b92fcf75ba3 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/8f5728c3f59d4b18a2f56b92fcf75ba3 2024-11-26T16:43:04,593 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/d6c9578eb3b44efc8c07be6ce20e63e5 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/d6c9578eb3b44efc8c07be6ce20e63e5 2024-11-26T16:43:04,596 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/B of 42f6c759eca28a5c6a511cb6fa26803c into d6c9578eb3b44efc8c07be6ce20e63e5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:43:04,596 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/A of 42f6c759eca28a5c6a511cb6fa26803c into 8f5728c3f59d4b18a2f56b92fcf75ba3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:43:04,596 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:43:04,596 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:43:04,596 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/B, priority=12, startTime=1732639384175; duration=0sec 2024-11-26T16:43:04,596 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/A, priority=12, startTime=1732639384175; duration=0sec 2024-11-26T16:43:04,596 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:43:04,597 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:B 2024-11-26T16:43:04,597 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:43:04,597 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:A 2024-11-26T16:43:04,597 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:43:04,597 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47578 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:43:04,597 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): 42f6c759eca28a5c6a511cb6fa26803c/C is initiating minor compaction (all files) 2024-11-26T16:43:04,598 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42f6c759eca28a5c6a511cb6fa26803c/C in TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:43:04,598 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/3ec95ae6fab44764846450ff6e2eb699, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/51d6c0c457634aa2ab0cca6d4cf6f17b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/65be6f88b5164b50aa89588634969405, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a4d343cf1c1c422a974cd83c617a6bef] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp, totalSize=46.5 K 2024-11-26T16:43:04,598 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ec95ae6fab44764846450ff6e2eb699, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1732639378137 2024-11-26T16:43:04,598 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 51d6c0c457634aa2ab0cca6d4cf6f17b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=630, earliestPutTs=1732639378698 2024-11-26T16:43:04,598 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 65be6f88b5164b50aa89588634969405, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=638, earliestPutTs=1732639379894 2024-11-26T16:43:04,598 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a4d343cf1c1c422a974cd83c617a6bef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=653, earliestPutTs=1732639382100 2024-11-26T16:43:04,605 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42f6c759eca28a5c6a511cb6fa26803c#C#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:43:04,605 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/6742e6d7248245a4b5ee3afa9e61f5e2 is 50, key is test_row_0/C:col10/1732639382931/Put/seqid=0 2024-11-26T16:43:04,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742344_1520 (size=13255) 2024-11-26T16:43:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T16:43:04,685 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(122): Close 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:43:04,685 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T16:43:04,685 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1722): Closing 42f6c759eca28a5c6a511cb6fa26803c, disabling compactions & flushes 2024-11-26T16:43:04,685 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:43:04,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T16:43:05,012 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/6742e6d7248245a4b5ee3afa9e61f5e2 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/6742e6d7248245a4b5ee3afa9e61f5e2 2024-11-26T16:43:05,016 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42f6c759eca28a5c6a511cb6fa26803c/C of 42f6c759eca28a5c6a511cb6fa26803c into 6742e6d7248245a4b5ee3afa9e61f5e2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:43:05,016 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42f6c759eca28a5c6a511cb6fa26803c: 2024-11-26T16:43:05,016 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:43:05,016 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., storeName=42f6c759eca28a5c6a511cb6fa26803c/C, priority=12, startTime=1732639384175; duration=0sec 2024-11-26T16:43:05,016 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:43:05,016 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:05,016 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. after waiting 0 ms 2024-11-26T16:43:05,016 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42f6c759eca28a5c6a511cb6fa26803c:C 2024-11-26T16:43:05,016 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:43:05,016 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(2902): Flushing 42f6c759eca28a5c6a511cb6fa26803c 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-26T16:43:05,017 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=A 2024-11-26T16:43:05,017 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:05,017 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=B 2024-11-26T16:43:05,017 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:05,017 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42f6c759eca28a5c6a511cb6fa26803c, store=C 2024-11-26T16:43:05,017 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:05,020 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/5b69f3eecfa644cda0f1e338f34d8493 is 50, key is test_row_2/A:col10/1732639384522/Put/seqid=0 2024-11-26T16:43:05,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742345_1521 (size=7415) 2024-11-26T16:43:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T16:43:05,423 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=661 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/5b69f3eecfa644cda0f1e338f34d8493 2024-11-26T16:43:05,429 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/060fd04d5ee846399f4a0daf4ea65af6 is 50, key is test_row_2/B:col10/1732639384522/Put/seqid=0 2024-11-26T16:43:05,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742346_1522 (size=7415) 2024-11-26T16:43:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T16:43:05,833 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=661 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/060fd04d5ee846399f4a0daf4ea65af6 2024-11-26T16:43:05,838 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/93431bb325fe4ae08a18c66a040d48c9 is 50, key is test_row_2/C:col10/1732639384522/Put/seqid=0 2024-11-26T16:43:05,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742347_1523 (size=7415) 2024-11-26T16:43:06,242 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=661 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/93431bb325fe4ae08a18c66a040d48c9 2024-11-26T16:43:06,246 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/A/5b69f3eecfa644cda0f1e338f34d8493 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/5b69f3eecfa644cda0f1e338f34d8493 2024-11-26T16:43:06,249 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/5b69f3eecfa644cda0f1e338f34d8493, entries=50, sequenceid=661, filesize=7.2 K 2024-11-26T16:43:06,249 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/B/060fd04d5ee846399f4a0daf4ea65af6 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/060fd04d5ee846399f4a0daf4ea65af6 2024-11-26T16:43:06,252 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/060fd04d5ee846399f4a0daf4ea65af6, entries=50, sequenceid=661, filesize=7.2 K 2024-11-26T16:43:06,252 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/.tmp/C/93431bb325fe4ae08a18c66a040d48c9 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/93431bb325fe4ae08a18c66a040d48c9 2024-11-26T16:43:06,255 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/93431bb325fe4ae08a18c66a040d48c9, entries=50, sequenceid=661, filesize=7.2 K 2024-11-26T16:43:06,256 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 42f6c759eca28a5c6a511cb6fa26803c in 1239ms, sequenceid=661, compaction requested=false 2024-11-26T16:43:06,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c644c2638e814089a732700ee200b34e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/9889a0c1ebd7454793e3528034a07686, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/e00883caa2384888befef8042ed6a636, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/90aaf67a7bc048ef9d6f5bac4f124667, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a056b30325c94a7c883300f8ffbb779b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/55ba2e99fd1f46e29ed2c8a1db33b81d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/b7f28d35561d4a168279864e62acdfc2, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a6db0da0f0a74bc8b57eabf4749937ed, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a79723cf49ad4a5ba7c6923e92a7a3b8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/7093fbf8ad474107b6ddaa1f859c8a0e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/6392ee7993ca407d9f979715ad7bff9e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/17e46581743f484687fd0200712d0069, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/7c585aa0d209472484be99b1ca2aaa80, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/844b8bcee6a543938a18d23f1713b7cc, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/2ce3ef96333448e2b8d266851e7d78ca, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/33abd540d1164ce58b56d87ee869dbc4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/e74285539b3440a39ec242857a633139, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/af8c6799e1e641429e866c101602226b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/9e052426db0048c2a7fdea80dd669d4f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/8859c4624a144e34924ffc7dca06c3c6, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c74ea5f927eb4ec39a13a8ade3aabd8a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/8be02f63ce764bd7af2d17545aadd27e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a6b606eb65e84c1ea054e558eb433190, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c9ad89990db244468bd77fd76c7835ce, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/04a7971484f34cd7999618b036fa5d64, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c508f365e8694c28801ecbf418c0a4c3, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/8c7ccfe2f6dc412f9f7889773b76d1b1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/fbed778084084a3aa0140d96e27400ef] to archive 2024-11-26T16:43:06,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T16:43:06,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c644c2638e814089a732700ee200b34e to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c644c2638e814089a732700ee200b34e 2024-11-26T16:43:06,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/9889a0c1ebd7454793e3528034a07686 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/9889a0c1ebd7454793e3528034a07686 2024-11-26T16:43:06,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/e00883caa2384888befef8042ed6a636 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/e00883caa2384888befef8042ed6a636 2024-11-26T16:43:06,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/90aaf67a7bc048ef9d6f5bac4f124667 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/90aaf67a7bc048ef9d6f5bac4f124667 2024-11-26T16:43:06,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a056b30325c94a7c883300f8ffbb779b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a056b30325c94a7c883300f8ffbb779b 2024-11-26T16:43:06,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/55ba2e99fd1f46e29ed2c8a1db33b81d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/55ba2e99fd1f46e29ed2c8a1db33b81d 2024-11-26T16:43:06,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/b7f28d35561d4a168279864e62acdfc2 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/b7f28d35561d4a168279864e62acdfc2 2024-11-26T16:43:06,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a6db0da0f0a74bc8b57eabf4749937ed to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a6db0da0f0a74bc8b57eabf4749937ed 2024-11-26T16:43:06,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a79723cf49ad4a5ba7c6923e92a7a3b8 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a79723cf49ad4a5ba7c6923e92a7a3b8 2024-11-26T16:43:06,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/7093fbf8ad474107b6ddaa1f859c8a0e to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/7093fbf8ad474107b6ddaa1f859c8a0e 2024-11-26T16:43:06,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/6392ee7993ca407d9f979715ad7bff9e to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/6392ee7993ca407d9f979715ad7bff9e 2024-11-26T16:43:06,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/17e46581743f484687fd0200712d0069 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/17e46581743f484687fd0200712d0069 2024-11-26T16:43:06,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/7c585aa0d209472484be99b1ca2aaa80 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/7c585aa0d209472484be99b1ca2aaa80 2024-11-26T16:43:06,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/844b8bcee6a543938a18d23f1713b7cc to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/844b8bcee6a543938a18d23f1713b7cc 2024-11-26T16:43:06,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/2ce3ef96333448e2b8d266851e7d78ca to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/2ce3ef96333448e2b8d266851e7d78ca 2024-11-26T16:43:06,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/33abd540d1164ce58b56d87ee869dbc4 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/33abd540d1164ce58b56d87ee869dbc4 2024-11-26T16:43:06,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/e74285539b3440a39ec242857a633139 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/e74285539b3440a39ec242857a633139 2024-11-26T16:43:06,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/af8c6799e1e641429e866c101602226b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/af8c6799e1e641429e866c101602226b 2024-11-26T16:43:06,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/9e052426db0048c2a7fdea80dd669d4f to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/9e052426db0048c2a7fdea80dd669d4f 2024-11-26T16:43:06,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/8859c4624a144e34924ffc7dca06c3c6 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/8859c4624a144e34924ffc7dca06c3c6 2024-11-26T16:43:06,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c74ea5f927eb4ec39a13a8ade3aabd8a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c74ea5f927eb4ec39a13a8ade3aabd8a 2024-11-26T16:43:06,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/8be02f63ce764bd7af2d17545aadd27e to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/8be02f63ce764bd7af2d17545aadd27e 2024-11-26T16:43:06,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a6b606eb65e84c1ea054e558eb433190 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/a6b606eb65e84c1ea054e558eb433190 2024-11-26T16:43:06,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c9ad89990db244468bd77fd76c7835ce to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c9ad89990db244468bd77fd76c7835ce 2024-11-26T16:43:06,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/04a7971484f34cd7999618b036fa5d64 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/04a7971484f34cd7999618b036fa5d64 2024-11-26T16:43:06,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c508f365e8694c28801ecbf418c0a4c3 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/c508f365e8694c28801ecbf418c0a4c3 2024-11-26T16:43:06,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/8c7ccfe2f6dc412f9f7889773b76d1b1 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/8c7ccfe2f6dc412f9f7889773b76d1b1 2024-11-26T16:43:06,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/fbed778084084a3aa0140d96e27400ef to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/fbed778084084a3aa0140d96e27400ef 2024-11-26T16:43:06,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/23b8ad53230e4abba9bcb1882b897f00, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/b0fca830eeb64caea886a31105c06832, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/06a23e3b6bb04027bb4434046ceaceaa, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/0720ef26e1504bf08d714a88880d55ff, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/2a2d906fb66841d0a3d9dedbb2d67048, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/035e1139026e42a3ba8d02dc912b8f98, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/f3b42eeb55b5492889bfc3c5427d681f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/6ca83f4de45e4079b6f7a1ffce4d7498, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/1fec3270cbba45358c02922717b6b41e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/77d6596d57584ae393cd5966f04141bd, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/c8e971330edf4002ab8a501220c06f21, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/458f759690414d7d8a3d400e4c1b4484, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/4378acfdc44945fb956538b1b95629aa, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/65035fac72f04a1d92d89bbcc0e1b778, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/a86048bbd220425d84347673ff04e0c1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/9de3af1d2cad4ee4b815f1809e128f74, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/83575546c9fe409e943a11eee0bbf7eb, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/233e3ec1efc1451c8418e09adffea896, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/93f625bba304467cbc6a008cc1d36028, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/19a9416609a04c58a3668d38d11f1145, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/0da7c5cf5bf247988845b59439a086cf, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/d702d21cbea44a85b9d881db726e15df, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/6518704ad4164404b11da7393ef5db06, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/889fd3be3e5e4d51b8237eede32cfe67, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/7b7768b80f4543859e064a32ec10a5f6, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/2d6dd1cd400543e2b3ed229497d9572d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/80ed40a3753d4c42bd25bc23c3d14eff, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/b9283db3e45b4ae18d74771b65f0e0dd] to archive 2024-11-26T16:43:06,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T16:43:06,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/23b8ad53230e4abba9bcb1882b897f00 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/23b8ad53230e4abba9bcb1882b897f00 2024-11-26T16:43:06,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/b0fca830eeb64caea886a31105c06832 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/b0fca830eeb64caea886a31105c06832 2024-11-26T16:43:06,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/06a23e3b6bb04027bb4434046ceaceaa to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/06a23e3b6bb04027bb4434046ceaceaa 2024-11-26T16:43:06,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/0720ef26e1504bf08d714a88880d55ff to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/0720ef26e1504bf08d714a88880d55ff 2024-11-26T16:43:06,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/2a2d906fb66841d0a3d9dedbb2d67048 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/2a2d906fb66841d0a3d9dedbb2d67048 2024-11-26T16:43:06,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/035e1139026e42a3ba8d02dc912b8f98 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/035e1139026e42a3ba8d02dc912b8f98 2024-11-26T16:43:06,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/f3b42eeb55b5492889bfc3c5427d681f to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/f3b42eeb55b5492889bfc3c5427d681f 2024-11-26T16:43:06,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/6ca83f4de45e4079b6f7a1ffce4d7498 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/6ca83f4de45e4079b6f7a1ffce4d7498 2024-11-26T16:43:06,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/1fec3270cbba45358c02922717b6b41e to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/1fec3270cbba45358c02922717b6b41e 2024-11-26T16:43:06,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/77d6596d57584ae393cd5966f04141bd to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/77d6596d57584ae393cd5966f04141bd 2024-11-26T16:43:06,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/c8e971330edf4002ab8a501220c06f21 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/c8e971330edf4002ab8a501220c06f21 2024-11-26T16:43:06,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/458f759690414d7d8a3d400e4c1b4484 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/458f759690414d7d8a3d400e4c1b4484 2024-11-26T16:43:06,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/4378acfdc44945fb956538b1b95629aa to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/4378acfdc44945fb956538b1b95629aa 2024-11-26T16:43:06,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/65035fac72f04a1d92d89bbcc0e1b778 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/65035fac72f04a1d92d89bbcc0e1b778 2024-11-26T16:43:06,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/a86048bbd220425d84347673ff04e0c1 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/a86048bbd220425d84347673ff04e0c1 2024-11-26T16:43:06,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/9de3af1d2cad4ee4b815f1809e128f74 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/9de3af1d2cad4ee4b815f1809e128f74 2024-11-26T16:43:06,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/83575546c9fe409e943a11eee0bbf7eb to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/83575546c9fe409e943a11eee0bbf7eb 2024-11-26T16:43:06,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/233e3ec1efc1451c8418e09adffea896 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/233e3ec1efc1451c8418e09adffea896 2024-11-26T16:43:06,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/93f625bba304467cbc6a008cc1d36028 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/93f625bba304467cbc6a008cc1d36028 2024-11-26T16:43:06,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/19a9416609a04c58a3668d38d11f1145 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/19a9416609a04c58a3668d38d11f1145 2024-11-26T16:43:06,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/0da7c5cf5bf247988845b59439a086cf to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/0da7c5cf5bf247988845b59439a086cf 2024-11-26T16:43:06,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/d702d21cbea44a85b9d881db726e15df to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/d702d21cbea44a85b9d881db726e15df 2024-11-26T16:43:06,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/6518704ad4164404b11da7393ef5db06 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/6518704ad4164404b11da7393ef5db06 2024-11-26T16:43:06,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/889fd3be3e5e4d51b8237eede32cfe67 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/889fd3be3e5e4d51b8237eede32cfe67 2024-11-26T16:43:06,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/7b7768b80f4543859e064a32ec10a5f6 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/7b7768b80f4543859e064a32ec10a5f6 2024-11-26T16:43:06,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/2d6dd1cd400543e2b3ed229497d9572d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/2d6dd1cd400543e2b3ed229497d9572d 2024-11-26T16:43:06,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/80ed40a3753d4c42bd25bc23c3d14eff to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/80ed40a3753d4c42bd25bc23c3d14eff 2024-11-26T16:43:06,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/b9283db3e45b4ae18d74771b65f0e0dd to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/b9283db3e45b4ae18d74771b65f0e0dd 2024-11-26T16:43:06,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/987ec28dd7754bf3bb652f7761a740ee, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/75d6789ffe7c40f3b9e29eb385a8ced9, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/56992abf7728400a8c888d7e37ec1aea, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/05cbb6de2f2c4701a14b18bb65172ae8, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/344315cc63da4c58ac0f46fde313bc9a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/0b5898f264e44d80bce0ce389dc751e7, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a3aaa8637e58488faf5368c6522feb09, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/8f5583bccc7a4f20b6310ef7d4ded3fa, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/d196a73e15f847cbac1b321bfc533c9a, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/259dedb66e2b419b82d983866a375231, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/5c4e5ce368b748a4aa728da71e82f90b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/0537a21c170b4f75b387637639709df9, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/117a7871cadd4d04a906aa56659682d5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a04394af98e747b2a7bc678a32ac9624, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/28755bd78fba4b889b85f1661eab1ad5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/0a901b0da82e4b37bd41ec5320e1c585, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/aa277131ae0141298bce024ef6fd6fe3, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/1400b9993c2b44bea2e440e57f69a0b5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/c515110182ce406d94563b8f8de5d093, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a0bf18bb789e4ff0a6c6460b8b0b3a01, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/d8214eb5d3154112bddfb3db3e5cfa90, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/b0da9c9a7a354463883a05629a900bae, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/ea92a6753cf94506b340ba6981759ffe, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/3ec95ae6fab44764846450ff6e2eb699, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/5c82f222f14145cf87799e64f3730dd0, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/51d6c0c457634aa2ab0cca6d4cf6f17b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/65be6f88b5164b50aa89588634969405, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a4d343cf1c1c422a974cd83c617a6bef] to archive 2024-11-26T16:43:06,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T16:43:06,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/987ec28dd7754bf3bb652f7761a740ee to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/987ec28dd7754bf3bb652f7761a740ee 2024-11-26T16:43:06,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/75d6789ffe7c40f3b9e29eb385a8ced9 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/75d6789ffe7c40f3b9e29eb385a8ced9 2024-11-26T16:43:06,305 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T16:43:06,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/56992abf7728400a8c888d7e37ec1aea to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/56992abf7728400a8c888d7e37ec1aea 2024-11-26T16:43:06,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/05cbb6de2f2c4701a14b18bb65172ae8 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/05cbb6de2f2c4701a14b18bb65172ae8 2024-11-26T16:43:06,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/344315cc63da4c58ac0f46fde313bc9a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/344315cc63da4c58ac0f46fde313bc9a 2024-11-26T16:43:06,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/0b5898f264e44d80bce0ce389dc751e7 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/0b5898f264e44d80bce0ce389dc751e7 2024-11-26T16:43:06,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a3aaa8637e58488faf5368c6522feb09 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a3aaa8637e58488faf5368c6522feb09 2024-11-26T16:43:06,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/8f5583bccc7a4f20b6310ef7d4ded3fa to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/8f5583bccc7a4f20b6310ef7d4ded3fa 2024-11-26T16:43:06,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/d196a73e15f847cbac1b321bfc533c9a to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/d196a73e15f847cbac1b321bfc533c9a 2024-11-26T16:43:06,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/259dedb66e2b419b82d983866a375231 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/259dedb66e2b419b82d983866a375231 2024-11-26T16:43:06,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/5c4e5ce368b748a4aa728da71e82f90b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/5c4e5ce368b748a4aa728da71e82f90b 2024-11-26T16:43:06,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/0537a21c170b4f75b387637639709df9 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/0537a21c170b4f75b387637639709df9 2024-11-26T16:43:06,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/117a7871cadd4d04a906aa56659682d5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/117a7871cadd4d04a906aa56659682d5 2024-11-26T16:43:06,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a04394af98e747b2a7bc678a32ac9624 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a04394af98e747b2a7bc678a32ac9624 2024-11-26T16:43:06,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/28755bd78fba4b889b85f1661eab1ad5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/28755bd78fba4b889b85f1661eab1ad5 2024-11-26T16:43:06,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/0a901b0da82e4b37bd41ec5320e1c585 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/0a901b0da82e4b37bd41ec5320e1c585 2024-11-26T16:43:06,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/aa277131ae0141298bce024ef6fd6fe3 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/aa277131ae0141298bce024ef6fd6fe3 2024-11-26T16:43:06,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/1400b9993c2b44bea2e440e57f69a0b5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/1400b9993c2b44bea2e440e57f69a0b5 2024-11-26T16:43:06,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/c515110182ce406d94563b8f8de5d093 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/c515110182ce406d94563b8f8de5d093 2024-11-26T16:43:06,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a0bf18bb789e4ff0a6c6460b8b0b3a01 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a0bf18bb789e4ff0a6c6460b8b0b3a01 2024-11-26T16:43:06,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/d8214eb5d3154112bddfb3db3e5cfa90 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/d8214eb5d3154112bddfb3db3e5cfa90 2024-11-26T16:43:06,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/b0da9c9a7a354463883a05629a900bae to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/b0da9c9a7a354463883a05629a900bae 2024-11-26T16:43:06,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/ea92a6753cf94506b340ba6981759ffe to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/ea92a6753cf94506b340ba6981759ffe 2024-11-26T16:43:06,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/3ec95ae6fab44764846450ff6e2eb699 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/3ec95ae6fab44764846450ff6e2eb699 2024-11-26T16:43:06,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/5c82f222f14145cf87799e64f3730dd0 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/5c82f222f14145cf87799e64f3730dd0 2024-11-26T16:43:06,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/51d6c0c457634aa2ab0cca6d4cf6f17b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/51d6c0c457634aa2ab0cca6d4cf6f17b 2024-11-26T16:43:06,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/65be6f88b5164b50aa89588634969405 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/65be6f88b5164b50aa89588634969405 2024-11-26T16:43:06,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a4d343cf1c1c422a974cd83c617a6bef to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/a4d343cf1c1c422a974cd83c617a6bef 2024-11-26T16:43:06,331 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/recovered.edits/664.seqid, newMaxSeqId=664, maxSeqId=1 2024-11-26T16:43:06,331 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. 2024-11-26T16:43:06,332 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1676): Region close journal for 42f6c759eca28a5c6a511cb6fa26803c: Waiting for close lock at 1732639384685Running coprocessor pre-close hooks at 1732639384685Disabling compacts and flushes for region at 1732639384685Disabling writes for close at 1732639385016 (+331 ms)Obtaining lock to block concurrent updates at 1732639385016Preparing flush snapshotting stores in 42f6c759eca28a5c6a511cb6fa26803c at 1732639385016Finished memstore snapshotting TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732639385017 (+1 ms)Flushing stores of TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c. at 1732639385017Flushing 42f6c759eca28a5c6a511cb6fa26803c/A: creating writer at 1732639385017Flushing 42f6c759eca28a5c6a511cb6fa26803c/A: appending metadata at 1732639385019 (+2 ms)Flushing 42f6c759eca28a5c6a511cb6fa26803c/A: closing flushed file at 1732639385019Flushing 42f6c759eca28a5c6a511cb6fa26803c/B: creating writer at 1732639385427 (+408 ms)Flushing 42f6c759eca28a5c6a511cb6fa26803c/B: appending metadata at 1732639385429 (+2 ms)Flushing 42f6c759eca28a5c6a511cb6fa26803c/B: closing flushed file at 1732639385429Flushing 42f6c759eca28a5c6a511cb6fa26803c/C: creating writer at 1732639385836 (+407 ms)Flushing 42f6c759eca28a5c6a511cb6fa26803c/C: appending metadata at 1732639385838 (+2 ms)Flushing 42f6c759eca28a5c6a511cb6fa26803c/C: closing flushed file at 1732639385838Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a637fa4: reopening flushed file at 1732639386246 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ea90b98: reopening flushed file at 1732639386249 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@85effda: reopening flushed file at 1732639386252 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 42f6c759eca28a5c6a511cb6fa26803c in 1239ms, sequenceid=661, compaction requested=false at 1732639386256 (+4 ms)Writing region close event to WAL at 1732639386329 (+73 ms)Running coprocessor post-close hooks at 1732639386331 (+2 ms)Closed at 1732639386331 2024-11-26T16:43:06,333 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(157): Closed 42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:43:06,334 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=42f6c759eca28a5c6a511cb6fa26803c, regionState=CLOSED 2024-11-26T16:43:06,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 because future has completed 2024-11-26T16:43:06,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-26T16:43:06,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseRegionProcedure 42f6c759eca28a5c6a511cb6fa26803c, server=2dedfe7466e8,38109,1732639238134 in 1.8030 sec 2024-11-26T16:43:06,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-26T16:43:06,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42f6c759eca28a5c6a511cb6fa26803c, UNASSIGN in 1.8070 sec 2024-11-26T16:43:06,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-26T16:43:06,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8100 sec 2024-11-26T16:43:06,341 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639386341"}]},"ts":"1732639386341"} 2024-11-26T16:43:06,342 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T16:43:06,342 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T16:43:06,344 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8180 sec 2024-11-26T16:43:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-26T16:43:06,671 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:43:06,672 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T16:43:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:43:06,673 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:43:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T16:43:06,674 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=131, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:43:06,676 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:43:06,678 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A, FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B, FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C, FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/recovered.edits] 2024-11-26T16:43:06,680 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/5b69f3eecfa644cda0f1e338f34d8493 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/5b69f3eecfa644cda0f1e338f34d8493 2024-11-26T16:43:06,680 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/8f5728c3f59d4b18a2f56b92fcf75ba3 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/A/8f5728c3f59d4b18a2f56b92fcf75ba3 2024-11-26T16:43:06,682 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/060fd04d5ee846399f4a0daf4ea65af6 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/060fd04d5ee846399f4a0daf4ea65af6 2024-11-26T16:43:06,683 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/d6c9578eb3b44efc8c07be6ce20e63e5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/B/d6c9578eb3b44efc8c07be6ce20e63e5 2024-11-26T16:43:06,685 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/6742e6d7248245a4b5ee3afa9e61f5e2 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/6742e6d7248245a4b5ee3afa9e61f5e2 2024-11-26T16:43:06,685 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/93431bb325fe4ae08a18c66a040d48c9 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/C/93431bb325fe4ae08a18c66a040d48c9 2024-11-26T16:43:06,687 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/recovered.edits/664.seqid to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c/recovered.edits/664.seqid 2024-11-26T16:43:06,688 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/42f6c759eca28a5c6a511cb6fa26803c 2024-11-26T16:43:06,688 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T16:43:06,690 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=131, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:43:06,691 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T16:43:06,693 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T16:43:06,694 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=131, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:43:06,694 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T16:43:06,694 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732639386694"}]},"ts":"9223372036854775807"} 2024-11-26T16:43:06,696 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T16:43:06,696 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 42f6c759eca28a5c6a511cb6fa26803c, NAME => 'TestAcidGuarantees,,1732639359922.42f6c759eca28a5c6a511cb6fa26803c.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T16:43:06,696 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T16:43:06,696 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732639386696"}]},"ts":"9223372036854775807"} 2024-11-26T16:43:06,697 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T16:43:06,698 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=131, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:43:06,698 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 26 msec 2024-11-26T16:43:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-26T16:43:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T16:43:06,782 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:43:06,793 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=253 (was 253), OpenFileDescriptor=463 (was 465), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=414 (was 421), ProcessCount=11 (was 11), AvailableMemoryMB=5134 (was 5187) 2024-11-26T16:43:06,802 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=253, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=414, ProcessCount=11, AvailableMemoryMB=5133 2024-11-26T16:43:06,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T16:43:06,803 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T16:43:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-26T16:43:06,805 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T16:43:06,805 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:06,805 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 132 2024-11-26T16:43:06,806 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T16:43:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-26T16:43:06,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742348_1524 (size=1061) 2024-11-26T16:43:06,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-26T16:43:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-26T16:43:07,212 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711 2024-11-26T16:43:07,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742349_1525 (size=53) 2024-11-26T16:43:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-26T16:43:07,617 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T16:43:07,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c551e376aede51019d0df72805dd3c05, disabling compactions & flushes 2024-11-26T16:43:07,618 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:07,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:07,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. after waiting 0 ms 2024-11-26T16:43:07,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:07,618 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:07,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c551e376aede51019d0df72805dd3c05: Waiting for close lock at 1732639387618Disabling compacts and flushes for region at 1732639387618Disabling writes for close at 1732639387618Writing region close event to WAL at 1732639387618Closed at 1732639387618 2024-11-26T16:43:07,619 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T16:43:07,619 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732639387619"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732639387619"}]},"ts":"1732639387619"} 2024-11-26T16:43:07,621 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T16:43:07,621 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T16:43:07,621 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639387621"}]},"ts":"1732639387621"} 2024-11-26T16:43:07,622 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-26T16:43:07,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c551e376aede51019d0df72805dd3c05, ASSIGN}] 2024-11-26T16:43:07,623 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c551e376aede51019d0df72805dd3c05, ASSIGN 2024-11-26T16:43:07,624 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c551e376aede51019d0df72805dd3c05, ASSIGN; state=OFFLINE, location=2dedfe7466e8,38109,1732639238134; forceNewPlan=false, retain=false 2024-11-26T16:43:07,775 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=c551e376aede51019d0df72805dd3c05, regionState=OPENING, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:07,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c551e376aede51019d0df72805dd3c05, ASSIGN because future has completed 2024-11-26T16:43:07,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134}] 2024-11-26T16:43:07,781 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-26T16:43:07,931 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:07,931 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7752): Opening region: {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} 2024-11-26T16:43:07,931 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:07,931 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T16:43:07,931 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7794): checking encryption for c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:07,931 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7797): checking classloading for c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:07,932 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:07,933 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:43:07,934 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c551e376aede51019d0df72805dd3c05 columnFamilyName A 2024-11-26T16:43:07,934 DEBUG [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:07,934 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] regionserver.HStore(327): Store=c551e376aede51019d0df72805dd3c05/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:43:07,934 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:07,935 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:43:07,935 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c551e376aede51019d0df72805dd3c05 columnFamilyName B 2024-11-26T16:43:07,935 DEBUG [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:07,936 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] regionserver.HStore(327): Store=c551e376aede51019d0df72805dd3c05/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:43:07,936 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:07,936 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:43:07,936 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c551e376aede51019d0df72805dd3c05 columnFamilyName C 2024-11-26T16:43:07,936 DEBUG [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:07,937 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] regionserver.HStore(327): Store=c551e376aede51019d0df72805dd3c05/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:43:07,937 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:07,937 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1038): replaying wal for c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:07,937 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:07,938 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:07,938 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1048): stopping wal replay for c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:07,938 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1060): Cleaning up temporary data for c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:07,939 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T16:43:07,940 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1093): writing seq id for c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:07,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-26T16:43:07,942 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T16:43:07,942 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1114): Opened c551e376aede51019d0df72805dd3c05; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59803263, jitterRate=-0.10886193811893463}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T16:43:07,942 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:07,943 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1006): Region open journal for c551e376aede51019d0df72805dd3c05: Running coprocessor pre-open hook at 1732639387931Writing region info on filesystem at 1732639387931Initializing all the Stores at 1732639387932 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639387932Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639387932Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639387932Cleaning up temporary data from old regions at 1732639387938 (+6 ms)Running coprocessor post-open hooks at 1732639387942 (+4 ms)Region opened successfully at 1732639387942 2024-11-26T16:43:07,943 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., pid=134, masterSystemTime=1732639387928 2024-11-26T16:43:07,945 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:07,945 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:07,945 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=c551e376aede51019d0df72805dd3c05, regionState=OPEN, openSeqNum=2, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:07,946 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 because future has completed 2024-11-26T16:43:07,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-26T16:43:07,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; OpenRegionProcedure c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 in 171 msec 2024-11-26T16:43:07,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-26T16:43:07,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c551e376aede51019d0df72805dd3c05, ASSIGN in 325 msec 2024-11-26T16:43:07,950 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T16:43:07,950 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639387950"}]},"ts":"1732639387950"} 2024-11-26T16:43:07,951 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-26T16:43:07,952 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T16:43:07,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-26T16:43:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-26T16:43:08,951 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:43:08,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f53afef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:08,952 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:43:08,952 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:43:08,953 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:43:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:43:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:43:08,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10416a57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:08,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:43:08,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:43:08,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:08,955 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:43:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38475867, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:08,956 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:43:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:43:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:43:08,957 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:43:08,958 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T16:43:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 2dedfe7466e8,35233,1732639237322 2024-11-26T16:43:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7d003d01 2024-11-26T16:43:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T16:43:08,959 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T16:43:08,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-26T16:43:08,960 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T16:43:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-26T16:43:08,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742350_1526 (size=1097) 2024-11-26T16:43:09,371 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-26T16:43:09,371 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-26T16:43:09,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T16:43:09,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c551e376aede51019d0df72805dd3c05, REOPEN/MOVE}] 2024-11-26T16:43:09,375 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c551e376aede51019d0df72805dd3c05, REOPEN/MOVE 2024-11-26T16:43:09,376 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=c551e376aede51019d0df72805dd3c05, regionState=CLOSING, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:09,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c551e376aede51019d0df72805dd3c05, REOPEN/MOVE because future has completed 2024-11-26T16:43:09,377 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T16:43:09,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134}] 2024-11-26T16:43:09,528 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:09,529 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T16:43:09,529 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing c551e376aede51019d0df72805dd3c05, disabling compactions & flushes 2024-11-26T16:43:09,529 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:09,529 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:09,529 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. after waiting 0 ms 2024-11-26T16:43:09,529 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:09,532 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T16:43:09,532 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:09,533 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for c551e376aede51019d0df72805dd3c05: Waiting for close lock at 1732639389529Running coprocessor pre-close hooks at 1732639389529Disabling compacts and flushes for region at 1732639389529Disabling writes for close at 1732639389529Writing region close event to WAL at 1732639389529Running coprocessor post-close hooks at 1732639389532 (+3 ms)Closed at 1732639389532 2024-11-26T16:43:09,533 WARN [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionServer(3299): Not adding moved region record: c551e376aede51019d0df72805dd3c05 to self. 2024-11-26T16:43:09,534 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:09,535 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=c551e376aede51019d0df72805dd3c05, regionState=CLOSED 2024-11-26T16:43:09,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 because future has completed 2024-11-26T16:43:09,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-26T16:43:09,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 in 160 msec 2024-11-26T16:43:09,539 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c551e376aede51019d0df72805dd3c05, REOPEN/MOVE; state=CLOSED, location=2dedfe7466e8,38109,1732639238134; forceNewPlan=false, retain=true 2024-11-26T16:43:09,689 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=c551e376aede51019d0df72805dd3c05, regionState=OPENING, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:09,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c551e376aede51019d0df72805dd3c05, REOPEN/MOVE because future has completed 2024-11-26T16:43:09,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134}] 2024-11-26T16:43:09,845 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:09,846 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7752): Opening region: {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} 2024-11-26T16:43:09,846 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:09,846 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T16:43:09,846 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7794): checking encryption for c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:09,846 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7797): checking classloading for c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:09,847 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:09,848 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:43:09,848 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c551e376aede51019d0df72805dd3c05 columnFamilyName A 2024-11-26T16:43:09,849 DEBUG [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:09,849 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] regionserver.HStore(327): Store=c551e376aede51019d0df72805dd3c05/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:43:09,849 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:09,850 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:43:09,850 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c551e376aede51019d0df72805dd3c05 columnFamilyName B 2024-11-26T16:43:09,850 DEBUG [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:09,850 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] regionserver.HStore(327): Store=c551e376aede51019d0df72805dd3c05/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:43:09,850 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:09,851 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-26T16:43:09,851 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c551e376aede51019d0df72805dd3c05 columnFamilyName C 2024-11-26T16:43:09,851 DEBUG [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:09,851 INFO [StoreOpener-c551e376aede51019d0df72805dd3c05-1 {}] regionserver.HStore(327): Store=c551e376aede51019d0df72805dd3c05/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T16:43:09,851 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:09,851 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1038): replaying wal for c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:09,852 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:09,853 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:09,853 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1048): stopping wal replay for c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:09,853 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1060): Cleaning up temporary data for c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:09,854 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T16:43:09,855 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1093): writing seq id for c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:09,855 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1114): Opened c551e376aede51019d0df72805dd3c05; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62646827, jitterRate=-0.06648953258991241}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T16:43:09,855 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:09,856 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1006): Region open journal for c551e376aede51019d0df72805dd3c05: Running coprocessor pre-open hook at 1732639389846Writing region info on filesystem at 1732639389846Initializing all the Stores at 1732639389847 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639389847Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639389847Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732639389847Cleaning up temporary data from old regions at 1732639389853 (+6 ms)Running coprocessor post-open hooks at 1732639389855 (+2 ms)Region opened successfully at 1732639389856 (+1 ms) 2024-11-26T16:43:09,857 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., pid=139, masterSystemTime=1732639389843 2024-11-26T16:43:09,859 DEBUG [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:09,859 INFO [RS_OPEN_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:09,859 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=c551e376aede51019d0df72805dd3c05, regionState=OPEN, openSeqNum=5, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:09,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 because future has completed 2024-11-26T16:43:09,862 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=137 2024-11-26T16:43:09,862 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 in 170 msec 2024-11-26T16:43:09,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-26T16:43:09,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c551e376aede51019d0df72805dd3c05, REOPEN/MOVE in 487 msec 2024-11-26T16:43:09,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-26T16:43:09,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-11-26T16:43:09,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 905 msec 2024-11-26T16:43:09,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-26T16:43:09,868 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:43:09,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3071d898, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,869 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:43:09,869 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:43:09,870 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:43:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:43:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:43:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c22b5d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:43:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:43:09,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:09,871 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:43:09,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11865991, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a209286, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,872 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:43:09,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:43:09,873 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:43:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:43:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:43:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68372ce8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:43:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:43:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:09,874 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:43:09,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cc5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48fa0b51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,875 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:43:09,875 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:43:09,875 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:43:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:43:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:43:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@159c762b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:43:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:43:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:09,877 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:43:09,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c83c39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1130c997, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:43:09,878 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:43:09,878 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:43:09,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:43:09,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:43:09,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a317faa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:43:09,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:43:09,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:09,880 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:43:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ac2361, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29acc1f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:43:09,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:43:09,881 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:43:09,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:43:09,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:43:09,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75befc04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:43:09,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:43:09,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:09,882 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:43:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@614da0b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d301717, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,883 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:43:09,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:43:09,884 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:43:09,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:43:09,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:43:09,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@edb020e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:43:09,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:43:09,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:09,885 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:43:09,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@357a23bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36b0a686, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,886 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:43:09,886 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:43:09,887 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:43:09,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:43:09,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:43:09,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e346c5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:43:09,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:43:09,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:09,888 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:43:09,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3df68206, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18d05b81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:43:09,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:43:09,889 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:43:09,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:43:09,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:43:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d68da98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:43:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:43:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:09,890 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:43:09,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b8e1a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,891 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27def64b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,891 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:43:09,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:43:09,892 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:43:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:43:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:43:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d395a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:43:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:43:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:09,893 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:43:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@178345a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37c440df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2dedfe7466e8,35233,-1 for getting cluster id 2024-11-26T16:43:09,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T16:43:09,894 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0d93ae86-e698-43e8-8f2c-76f242eed8e3' 2024-11-26T16:43:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T16:43:09,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0d93ae86-e698-43e8-8f2c-76f242eed8e3" 2024-11-26T16:43:09,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49d9fc87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2dedfe7466e8,35233,-1] 2024-11-26T16:43:09,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T16:43:09,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:09,895 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T16:43:09,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f78a120, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T16:43:09,896 DEBUG [Thread-2350 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:43:09,896 DEBUG [Thread-2361 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:43:09,896 DEBUG [Thread-2363 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:43:09,896 DEBUG [Thread-2354 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:43:09,896 DEBUG [Thread-2348 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:43:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:43:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:43:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:43:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:43:09,897 DEBUG [Thread-2367 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:43:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:43:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:43:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:43:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:43:09,897 DEBUG [Thread-2352 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:43:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:43:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:43:09,898 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:43:09,898 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:43:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:43:09,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:43:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:43:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:43:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:43:09,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:43:09,899 DEBUG [Thread-2359 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:43:09,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:43:09,900 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:43:09,900 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:43:09,900 DEBUG [Thread-2356 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:43:09,900 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:43:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:43:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:43:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:43:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:43:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:43:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:43:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:43:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:43:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:43:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:43:09,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:43:09,903 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:43:09,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:43:09,903 DEBUG [Thread-2365 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T16:43:09,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-26T16:43:09,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2dedfe7466e8,38109,1732639238134, seqNum=-1] 2024-11-26T16:43:09,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T16:43:09,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:43:09,905 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:43:09,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T16:43:09,905 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:43:09,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T16:43:09,906 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:43:09,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:43:09,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5] 2024-11-26T16:43:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:09,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c551e376aede51019d0df72805dd3c05 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-26T16:43:09,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=A 2024-11-26T16:43:09,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:09,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=B 2024-11-26T16:43:09,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:09,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=C 2024-11-26T16:43:09,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:09,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:09,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639449931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:09,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639449932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:09,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:09,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639449936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:09,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:09,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:09,941 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:09,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:09,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639449937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:09,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:09,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:09,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:09,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639449938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:09,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:09,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:09,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:09,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:09,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267978922de13b48b9866b443d44e47a58_c551e376aede51019d0df72805dd3c05 is 50, key is test_row_0/A:col10/1732639389906/Put/seqid=0 2024-11-26T16:43:09,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742351_1527 (size=12154) 2024-11-26T16:43:09,955 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:09,958 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267978922de13b48b9866b443d44e47a58_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267978922de13b48b9866b443d44e47a58_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:09,959 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/6e36f0320b644d24820f586e5dba8f40, store: [table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:09,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/6e36f0320b644d24820f586e5dba8f40 is 175, key is test_row_0/A:col10/1732639389906/Put/seqid=0 2024-11-26T16:43:09,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742352_1528 (size=30955) 2024-11-26T16:43:09,965 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/6e36f0320b644d24820f586e5dba8f40 2024-11-26T16:43:09,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/ae694fbec5e04991946f23fb4088d7f4 is 50, key is test_row_0/B:col10/1732639389906/Put/seqid=0 2024-11-26T16:43:09,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742353_1529 (size=12001) 2024-11-26T16:43:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T16:43:10,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639450041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:10,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639450041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:10,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:10,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639450051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:10,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:10,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639450051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:10,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:10,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639450051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:10,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:10,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T16:43:10,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:10,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:10,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:10,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:10,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:10,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T16:43:10,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:10,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:10,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:10,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:10,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T16:43:10,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639450251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:10,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639450251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:10,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:10,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:10,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639450261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:10,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639450261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639450261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:10,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T16:43:10,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:10,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:10,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:10,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:10,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:10,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/ae694fbec5e04991946f23fb4088d7f4 2024-11-26T16:43:10,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/f5f112ddfe974376ba0b782d7f224a26 is 50, key is test_row_0/C:col10/1732639389906/Put/seqid=0 2024-11-26T16:43:10,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742354_1530 (size=12001) 2024-11-26T16:43:10,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T16:43:10,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:10,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:10,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:10,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:10,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:10,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T16:43:10,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639450561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:10,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639450561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:10,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:10,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639450571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:10,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:10,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639450571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:10,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639450571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:10,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:10,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:10,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:10,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T16:43:10,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:10,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:10,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:10,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:10,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:10,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:10,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/f5f112ddfe974376ba0b782d7f224a26 2024-11-26T16:43:10,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/6e36f0320b644d24820f586e5dba8f40 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/6e36f0320b644d24820f586e5dba8f40 2024-11-26T16:43:10,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T16:43:10,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:10,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:10,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:10,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:10,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:10,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:10,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/6e36f0320b644d24820f586e5dba8f40, entries=150, sequenceid=16, filesize=30.2 K 2024-11-26T16:43:10,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/ae694fbec5e04991946f23fb4088d7f4 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/ae694fbec5e04991946f23fb4088d7f4 2024-11-26T16:43:10,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/ae694fbec5e04991946f23fb4088d7f4, entries=150, sequenceid=16, filesize=11.7 K 2024-11-26T16:43:10,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/f5f112ddfe974376ba0b782d7f224a26 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/f5f112ddfe974376ba0b782d7f224a26 2024-11-26T16:43:10,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/f5f112ddfe974376ba0b782d7f224a26, entries=150, sequenceid=16, filesize=11.7 K 2024-11-26T16:43:10,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c551e376aede51019d0df72805dd3c05 in 928ms, sequenceid=16, compaction requested=false 2024-11-26T16:43:10,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:10,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-26T16:43:10,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:10,979 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing c551e376aede51019d0df72805dd3c05 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T16:43:10,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=A 2024-11-26T16:43:10,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:10,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=B 2024-11-26T16:43:10,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:10,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=C 2024-11-26T16:43:10,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:10,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e486f236cd584e4b92bf1af7422a0803_c551e376aede51019d0df72805dd3c05 is 50, key is test_row_0/A:col10/1732639389936/Put/seqid=0 2024-11-26T16:43:10,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742355_1531 (size=12154) 2024-11-26T16:43:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T16:43:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:11,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:11,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639451085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:11,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:11,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639451085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639451086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:11,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639451087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:11,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639451089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:11,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639451201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:11,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639451201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:11,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639451201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:11,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639451201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:11,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639451201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:11,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:11,395 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126e486f236cd584e4b92bf1af7422a0803_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e486f236cd584e4b92bf1af7422a0803_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:11,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/c9c983f4b81d47668d8506091410d9f5, store: [table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:11,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/c9c983f4b81d47668d8506091410d9f5 is 175, key is test_row_0/A:col10/1732639389936/Put/seqid=0 2024-11-26T16:43:11,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742356_1532 (size=30955) 2024-11-26T16:43:11,410 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/c9c983f4b81d47668d8506091410d9f5 2024-11-26T16:43:11,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639451411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:11,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639451411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:11,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:11,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639451412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:11,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:11,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639451412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:11,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639451412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:11,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:11,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/7a18a615d6df42b4a6929eafa4942a7c is 50, key is test_row_0/B:col10/1732639389936/Put/seqid=0 2024-11-26T16:43:11,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742357_1533 (size=12001) 2024-11-26T16:43:11,421 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/7a18a615d6df42b4a6929eafa4942a7c 2024-11-26T16:43:11,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/0522e8a2c6294d84a0cd3dc5d1e401e2 is 50, key is test_row_0/C:col10/1732639389936/Put/seqid=0 2024-11-26T16:43:11,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742358_1534 (size=12001) 2024-11-26T16:43:11,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639451721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:11,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:11,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639451721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:11,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:11,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:11,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639451721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:11,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639451721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:11,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639451721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:11,832 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/0522e8a2c6294d84a0cd3dc5d1e401e2 2024-11-26T16:43:11,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/c9c983f4b81d47668d8506091410d9f5 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/c9c983f4b81d47668d8506091410d9f5 2024-11-26T16:43:11,840 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/c9c983f4b81d47668d8506091410d9f5, entries=150, sequenceid=36, filesize=30.2 K 2024-11-26T16:43:11,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/7a18a615d6df42b4a6929eafa4942a7c as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/7a18a615d6df42b4a6929eafa4942a7c 2024-11-26T16:43:11,844 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/7a18a615d6df42b4a6929eafa4942a7c, entries=150, sequenceid=36, filesize=11.7 K 2024-11-26T16:43:11,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/0522e8a2c6294d84a0cd3dc5d1e401e2 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/0522e8a2c6294d84a0cd3dc5d1e401e2 2024-11-26T16:43:11,847 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T16:43:11,856 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/0522e8a2c6294d84a0cd3dc5d1e401e2, entries=150, sequenceid=36, filesize=11.7 K 2024-11-26T16:43:11,857 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c551e376aede51019d0df72805dd3c05 in 878ms, sequenceid=36, compaction requested=false 2024-11-26T16:43:11,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:11,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:11,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-26T16:43:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-26T16:43:11,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-26T16:43:11,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9530 sec 2024-11-26T16:43:11,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 1.9620 sec 2024-11-26T16:43:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-26T16:43:12,051 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:43:12,052 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:43:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-26T16:43:12,054 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:43:12,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-26T16:43:12,054 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:43:12,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:43:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-26T16:43:12,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-26T16:43:12,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:12,207 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing c551e376aede51019d0df72805dd3c05 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T16:43:12,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=A 2024-11-26T16:43:12,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:12,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=B 2024-11-26T16:43:12,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:12,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=C 2024-11-26T16:43:12,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:12,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112636709fd3f12d44b0b5c97ff92e6e5856_c551e376aede51019d0df72805dd3c05 is 50, key is test_row_0/A:col10/1732639391085/Put/seqid=0 2024-11-26T16:43:12,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742359_1535 (size=12154) 2024-11-26T16:43:12,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:12,220 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112636709fd3f12d44b0b5c97ff92e6e5856_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112636709fd3f12d44b0b5c97ff92e6e5856_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:12,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/4e00ae5a046b41c68082fbb18e99b967, store: [table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:12,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/4e00ae5a046b41c68082fbb18e99b967 is 175, key is test_row_0/A:col10/1732639391085/Put/seqid=0 2024-11-26T16:43:12,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742360_1536 (size=30955) 2024-11-26T16:43:12,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:12,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:12,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639452248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:12,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639452249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:12,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639452250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:12,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639452252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:12,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639452253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:12,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:12,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639452361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:12,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639452361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:12,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639452361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:12,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:12,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:12,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:12,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639452362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:12,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:12,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639452362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:12,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:12,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-26T16:43:12,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639452570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:12,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:12,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639452571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:12,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:12,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639452571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:12,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:12,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:12,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639452571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:12,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639452571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:12,626 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/4e00ae5a046b41c68082fbb18e99b967 2024-11-26T16:43:12,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/30b28182fd9343ee9284180d44d12f51 is 50, key is test_row_0/B:col10/1732639391085/Put/seqid=0 2024-11-26T16:43:12,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742361_1537 (size=12001) 2024-11-26T16:43:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-26T16:43:12,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:12,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639452880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:12,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:12,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639452881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:12,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:12,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:12,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639452881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:12,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:12,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:12,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639452881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:12,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:12,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:12,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639452881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:12,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:13,037 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/30b28182fd9343ee9284180d44d12f51 2024-11-26T16:43:13,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/4497716c9b9e47dc95598aaa1e105ac9 is 50, key is test_row_0/C:col10/1732639391085/Put/seqid=0 2024-11-26T16:43:13,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742362_1538 (size=12001) 2024-11-26T16:43:13,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-26T16:43:13,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639453391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:13,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639453391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:13,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639453391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:13,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639453391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:13,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639453392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:13,448 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/4497716c9b9e47dc95598aaa1e105ac9 2024-11-26T16:43:13,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/4e00ae5a046b41c68082fbb18e99b967 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/4e00ae5a046b41c68082fbb18e99b967 2024-11-26T16:43:13,456 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/4e00ae5a046b41c68082fbb18e99b967, entries=150, sequenceid=48, filesize=30.2 K 2024-11-26T16:43:13,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/30b28182fd9343ee9284180d44d12f51 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/30b28182fd9343ee9284180d44d12f51 2024-11-26T16:43:13,460 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/30b28182fd9343ee9284180d44d12f51, entries=150, sequenceid=48, filesize=11.7 K 2024-11-26T16:43:13,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/4497716c9b9e47dc95598aaa1e105ac9 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/4497716c9b9e47dc95598aaa1e105ac9 2024-11-26T16:43:13,464 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/4497716c9b9e47dc95598aaa1e105ac9, entries=150, sequenceid=48, filesize=11.7 K 2024-11-26T16:43:13,465 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c551e376aede51019d0df72805dd3c05 in 1257ms, sequenceid=48, compaction requested=true 2024-11-26T16:43:13,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:13,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:13,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-26T16:43:13,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-26T16:43:13,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-26T16:43:13,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4110 sec 2024-11-26T16:43:13,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 1.4160 sec 2024-11-26T16:43:14,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-26T16:43:14,201 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:43:14,201 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:43:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-26T16:43:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-26T16:43:14,203 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:43:14,204 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:43:14,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:43:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-26T16:43:14,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-26T16:43:14,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:14,357 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing c551e376aede51019d0df72805dd3c05 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T16:43:14,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=A 2024-11-26T16:43:14,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:14,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=B 2024-11-26T16:43:14,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:14,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=C 2024-11-26T16:43:14,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:14,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112669afffe6cbbb4edaaa1c03d0572dae15_c551e376aede51019d0df72805dd3c05 is 50, key is test_row_0/A:col10/1732639392249/Put/seqid=0 2024-11-26T16:43:14,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742363_1539 (size=12154) 2024-11-26T16:43:14,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:14,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:14,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:14,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639454442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:14,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639454442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:14,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:14,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:14,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639454444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:14,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639454444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:14,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:14,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:14,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:14,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:14,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639454451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-26T16:43:14,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:14,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639454561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:14,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:14,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639454561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:14,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639454561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:14,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639454561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:14,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639454561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:14,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639454771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639454771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:14,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639454771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639454771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:14,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639454771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:14,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:14,785 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112669afffe6cbbb4edaaa1c03d0572dae15_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112669afffe6cbbb4edaaa1c03d0572dae15_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:14,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/7927dc17f3ad44268d45d25eeec99c80, store: [table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:14,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/7927dc17f3ad44268d45d25eeec99c80 is 175, key is test_row_0/A:col10/1732639392249/Put/seqid=0 2024-11-26T16:43:14,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742364_1540 (size=30955) 2024-11-26T16:43:14,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-26T16:43:15,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639455081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:15,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:15,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:15,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:15,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639455081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:15,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:15,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:15,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:15,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639455081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:15,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639455081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:15,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639455081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:15,190 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/7927dc17f3ad44268d45d25eeec99c80 2024-11-26T16:43:15,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/73b8c69f92a849368783b82e59ff58bb is 50, key is test_row_0/B:col10/1732639392249/Put/seqid=0 2024-11-26T16:43:15,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742365_1541 (size=12001) 2024-11-26T16:43:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-26T16:43:15,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639455591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:15,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639455591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:15,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639455592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:15,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639455592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:15,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639455592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:15,610 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/73b8c69f92a849368783b82e59ff58bb 2024-11-26T16:43:15,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/eb867314c5d34a87a49ebbd599774dc1 is 50, key is test_row_0/C:col10/1732639392249/Put/seqid=0 2024-11-26T16:43:15,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742366_1542 (size=12001) 2024-11-26T16:43:16,021 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/eb867314c5d34a87a49ebbd599774dc1 2024-11-26T16:43:16,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/7927dc17f3ad44268d45d25eeec99c80 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/7927dc17f3ad44268d45d25eeec99c80 2024-11-26T16:43:16,028 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/7927dc17f3ad44268d45d25eeec99c80, entries=150, sequenceid=68, filesize=30.2 K 2024-11-26T16:43:16,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/73b8c69f92a849368783b82e59ff58bb as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/73b8c69f92a849368783b82e59ff58bb 2024-11-26T16:43:16,032 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/73b8c69f92a849368783b82e59ff58bb, entries=150, sequenceid=68, filesize=11.7 K 2024-11-26T16:43:16,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/eb867314c5d34a87a49ebbd599774dc1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/eb867314c5d34a87a49ebbd599774dc1 2024-11-26T16:43:16,036 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/eb867314c5d34a87a49ebbd599774dc1, entries=150, sequenceid=68, filesize=11.7 K 2024-11-26T16:43:16,037 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c551e376aede51019d0df72805dd3c05 in 1680ms, sequenceid=68, compaction requested=true 2024-11-26T16:43:16,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:16,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:16,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-26T16:43:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-26T16:43:16,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-26T16:43:16,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8340 sec 2024-11-26T16:43:16,042 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 1.8400 sec 2024-11-26T16:43:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-26T16:43:16,351 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:43:16,352 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:43:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-26T16:43:16,354 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:43:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T16:43:16,354 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:43:16,354 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:43:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T16:43:16,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-26T16:43:16,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:16,507 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing c551e376aede51019d0df72805dd3c05 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T16:43:16,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=A 2024-11-26T16:43:16,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:16,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=B 2024-11-26T16:43:16,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:16,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=C 2024-11-26T16:43:16,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:16,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d67209aa590145a09a3034bf00b8b8d1_c551e376aede51019d0df72805dd3c05 is 50, key is test_row_0/A:col10/1732639394412/Put/seqid=0 2024-11-26T16:43:16,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742367_1543 (size=12154) 2024-11-26T16:43:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:16,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:16,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639456627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:16,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639456628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:16,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:16,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639456628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:16,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639456631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:16,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:16,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639456634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:16,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T16:43:16,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639456741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:16,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:16,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639456741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:16,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639456741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:16,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639456741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:16,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:16,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639456751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:16,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:16,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:16,929 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126d67209aa590145a09a3034bf00b8b8d1_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d67209aa590145a09a3034bf00b8b8d1_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:16,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/b5da16b950d8460fab0f78de95dae99d, store: [table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:16,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/b5da16b950d8460fab0f78de95dae99d is 175, key is test_row_0/A:col10/1732639394412/Put/seqid=0 2024-11-26T16:43:16,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742368_1544 (size=30955) 2024-11-26T16:43:16,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639456951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:16,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639456951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:16,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639456951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:16,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639456952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:16,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639456961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:16,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T16:43:17,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639457261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:17,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639457261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:17,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639457262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:17,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639457262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:17,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639457271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:17,334 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/b5da16b950d8460fab0f78de95dae99d 2024-11-26T16:43:17,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/f5accc85c88643b4ab2f6be6bec56a8b is 50, key is test_row_0/B:col10/1732639394412/Put/seqid=0 2024-11-26T16:43:17,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742369_1545 (size=12001) 2024-11-26T16:43:17,345 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/f5accc85c88643b4ab2f6be6bec56a8b 2024-11-26T16:43:17,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/e8d8c03626be40e49f5981b1c861ddc1 is 50, key is test_row_0/C:col10/1732639394412/Put/seqid=0 2024-11-26T16:43:17,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742370_1546 (size=12001) 2024-11-26T16:43:17,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T16:43:17,756 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/e8d8c03626be40e49f5981b1c861ddc1 2024-11-26T16:43:17,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/b5da16b950d8460fab0f78de95dae99d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/b5da16b950d8460fab0f78de95dae99d 2024-11-26T16:43:17,763 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/b5da16b950d8460fab0f78de95dae99d, entries=150, sequenceid=80, filesize=30.2 K 2024-11-26T16:43:17,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/f5accc85c88643b4ab2f6be6bec56a8b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/f5accc85c88643b4ab2f6be6bec56a8b 2024-11-26T16:43:17,767 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/f5accc85c88643b4ab2f6be6bec56a8b, entries=150, sequenceid=80, filesize=11.7 K 2024-11-26T16:43:17,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/e8d8c03626be40e49f5981b1c861ddc1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/e8d8c03626be40e49f5981b1c861ddc1 2024-11-26T16:43:17,771 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/e8d8c03626be40e49f5981b1c861ddc1, entries=150, sequenceid=80, filesize=11.7 K 2024-11-26T16:43:17,772 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c551e376aede51019d0df72805dd3c05 in 1265ms, sequenceid=80, compaction requested=true 2024-11-26T16:43:17,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:17,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:17,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-26T16:43:17,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-26T16:43:17,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:17,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c551e376aede51019d0df72805dd3c05 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-26T16:43:17,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=A 2024-11-26T16:43:17,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:43:17,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=B 2024-11-26T16:43:17,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:43:17,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=C 2024-11-26T16:43:17,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:43:17,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-26T16:43:17,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4200 sec 2024-11-26T16:43:17,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.4270 sec 2024-11-26T16:43:17,781 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-26T16:43:17,781 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-26T16:43:17,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b46ba5688a68465b8231b90e10e4995c_c551e376aede51019d0df72805dd3c05 is 50, key is test_row_0/A:col10/1732639397773/Put/seqid=0 2024-11-26T16:43:17,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742371_1547 (size=12154) 2024-11-26T16:43:17,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639457788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:17,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639457791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:17,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639457791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:17,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639457792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639457793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:17,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639457901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:17,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639457902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:17,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:17,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:17,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639457911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:17,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639457911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:17,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:17,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:17,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639457911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:17,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:17,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:17,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:18,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639458111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:18,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639458111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:18,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:18,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:18,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639458121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:18,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639458121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:18,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639458121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:18,193 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:18,196 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b46ba5688a68465b8231b90e10e4995c_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b46ba5688a68465b8231b90e10e4995c_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:18,197 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/ea87091fe38b4c479cfa27c56d01c979, store: [table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:18,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/ea87091fe38b4c479cfa27c56d01c979 is 175, key is test_row_0/A:col10/1732639397773/Put/seqid=0 2024-11-26T16:43:18,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742372_1548 (size=30955) 2024-11-26T16:43:18,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639458421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:18,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639458421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:18,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639458431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:18,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639458431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:18,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639458431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:18,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-26T16:43:18,501 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:43:18,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:43:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-26T16:43:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T16:43:18,504 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:43:18,504 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:43:18,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:43:18,602 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=101, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/ea87091fe38b4c479cfa27c56d01c979 2024-11-26T16:43:18,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/9b9069e69fab4a9d96e31c67c7b1498f is 50, key is test_row_0/B:col10/1732639397773/Put/seqid=0 2024-11-26T16:43:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T16:43:18,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742373_1549 (size=12001) 2024-11-26T16:43:18,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T16:43:18,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:18,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:18,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:18,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:18,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:18,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T16:43:18,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:18,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:18,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:18,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:18,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T16:43:18,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639458931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:18,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639458931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:18,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639458941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:18,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639458941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:18,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:18,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639458941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:18,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:18,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T16:43:18,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:18,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:18,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:18,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:18,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:19,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/9b9069e69fab4a9d96e31c67c7b1498f 2024-11-26T16:43:19,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/596328aaad9d4e4fbd3cfaa585be273d is 50, key is test_row_0/C:col10/1732639397773/Put/seqid=0 2024-11-26T16:43:19,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742374_1550 (size=12001) 2024-11-26T16:43:19,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T16:43:19,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:19,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:19,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:19,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:19,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T16:43:19,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T16:43:19,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:19,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:19,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:19,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:19,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:19,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T16:43:19,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:19,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:19,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:19,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:19,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:19,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/596328aaad9d4e4fbd3cfaa585be273d 2024-11-26T16:43:19,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/ea87091fe38b4c479cfa27c56d01c979 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/ea87091fe38b4c479cfa27c56d01c979 2024-11-26T16:43:19,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/ea87091fe38b4c479cfa27c56d01c979, entries=150, sequenceid=101, filesize=30.2 K 2024-11-26T16:43:19,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/9b9069e69fab4a9d96e31c67c7b1498f as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/9b9069e69fab4a9d96e31c67c7b1498f 2024-11-26T16:43:19,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/9b9069e69fab4a9d96e31c67c7b1498f, entries=150, sequenceid=101, filesize=11.7 K 2024-11-26T16:43:19,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/596328aaad9d4e4fbd3cfaa585be273d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/596328aaad9d4e4fbd3cfaa585be273d 2024-11-26T16:43:19,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/596328aaad9d4e4fbd3cfaa585be273d, entries=150, sequenceid=101, filesize=11.7 K 2024-11-26T16:43:19,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for c551e376aede51019d0df72805dd3c05 in 1665ms, sequenceid=101, compaction requested=true 2024-11-26T16:43:19,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:19,440 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-26T16:43:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c551e376aede51019d0df72805dd3c05:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:43:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c551e376aede51019d0df72805dd3c05:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:43:19,441 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-26T16:43:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c551e376aede51019d0df72805dd3c05:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:43:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:43:19,442 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 185730 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-26T16:43:19,442 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): c551e376aede51019d0df72805dd3c05/A is initiating minor compaction (all files) 2024-11-26T16:43:19,442 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72006 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-26T16:43:19,442 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c551e376aede51019d0df72805dd3c05/A in TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:19,442 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): c551e376aede51019d0df72805dd3c05/B is initiating minor compaction (all files) 2024-11-26T16:43:19,442 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/6e36f0320b644d24820f586e5dba8f40, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/c9c983f4b81d47668d8506091410d9f5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/4e00ae5a046b41c68082fbb18e99b967, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/7927dc17f3ad44268d45d25eeec99c80, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/b5da16b950d8460fab0f78de95dae99d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/ea87091fe38b4c479cfa27c56d01c979] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp, totalSize=181.4 K 2024-11-26T16:43:19,442 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c551e376aede51019d0df72805dd3c05/B in TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:19,442 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:19,442 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/ae694fbec5e04991946f23fb4088d7f4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/7a18a615d6df42b4a6929eafa4942a7c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/30b28182fd9343ee9284180d44d12f51, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/73b8c69f92a849368783b82e59ff58bb, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/f5accc85c88643b4ab2f6be6bec56a8b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/9b9069e69fab4a9d96e31c67c7b1498f] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp, totalSize=70.3 K 2024-11-26T16:43:19,442 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. files: [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/6e36f0320b644d24820f586e5dba8f40, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/c9c983f4b81d47668d8506091410d9f5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/4e00ae5a046b41c68082fbb18e99b967, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/7927dc17f3ad44268d45d25eeec99c80, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/b5da16b950d8460fab0f78de95dae99d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/ea87091fe38b4c479cfa27c56d01c979] 2024-11-26T16:43:19,443 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting ae694fbec5e04991946f23fb4088d7f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732639389906 2024-11-26T16:43:19,443 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e36f0320b644d24820f586e5dba8f40, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732639389906 2024-11-26T16:43:19,443 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a18a615d6df42b4a6929eafa4942a7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732639389929 2024-11-26T16:43:19,443 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9c983f4b81d47668d8506091410d9f5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732639389929 2024-11-26T16:43:19,444 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 30b28182fd9343ee9284180d44d12f51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732639391081 2024-11-26T16:43:19,444 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e00ae5a046b41c68082fbb18e99b967, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732639391081 2024-11-26T16:43:19,444 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 73b8c69f92a849368783b82e59ff58bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1732639392234 2024-11-26T16:43:19,444 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7927dc17f3ad44268d45d25eeec99c80, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1732639392234 2024-11-26T16:43:19,444 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting f5accc85c88643b4ab2f6be6bec56a8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732639394412 2024-11-26T16:43:19,444 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5da16b950d8460fab0f78de95dae99d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732639394412 2024-11-26T16:43:19,445 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b9069e69fab4a9d96e31c67c7b1498f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732639396626 2024-11-26T16:43:19,445 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea87091fe38b4c479cfa27c56d01c979, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732639396626 2024-11-26T16:43:19,452 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:19,454 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126f080b96b70bc41d7bee78a47d76d0ce7_c551e376aede51019d0df72805dd3c05 store=[table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:19,454 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c551e376aede51019d0df72805dd3c05#B#compaction#468 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:43:19,455 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/c26fd24209114532bfaa675cae280fcd is 50, key is test_row_0/B:col10/1732639397773/Put/seqid=0 2024-11-26T16:43:19,456 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126f080b96b70bc41d7bee78a47d76d0ce7_c551e376aede51019d0df72805dd3c05, store=[table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:19,457 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126f080b96b70bc41d7bee78a47d76d0ce7_c551e376aede51019d0df72805dd3c05 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:19,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742375_1551 (size=12207) 2024-11-26T16:43:19,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742376_1552 (size=4469) 2024-11-26T16:43:19,475 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c551e376aede51019d0df72805dd3c05#A#compaction#469 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:43:19,475 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/b8db8196b91943eba44ccf0cf73914fc is 175, key is test_row_0/A:col10/1732639397773/Put/seqid=0 2024-11-26T16:43:19,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742377_1553 (size=31161) 2024-11-26T16:43:19,483 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/b8db8196b91943eba44ccf0cf73914fc as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/b8db8196b91943eba44ccf0cf73914fc 2024-11-26T16:43:19,488 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in c551e376aede51019d0df72805dd3c05/A of c551e376aede51019d0df72805dd3c05 into b8db8196b91943eba44ccf0cf73914fc(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:43:19,488 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:19,488 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., storeName=c551e376aede51019d0df72805dd3c05/A, priority=10, startTime=1732639399440; duration=0sec 2024-11-26T16:43:19,488 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:43:19,488 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c551e376aede51019d0df72805dd3c05:A 2024-11-26T16:43:19,488 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-26T16:43:19,489 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72006 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-26T16:43:19,490 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): c551e376aede51019d0df72805dd3c05/C is initiating minor compaction (all files) 2024-11-26T16:43:19,490 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c551e376aede51019d0df72805dd3c05/C in TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:19,490 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/f5f112ddfe974376ba0b782d7f224a26, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/0522e8a2c6294d84a0cd3dc5d1e401e2, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/4497716c9b9e47dc95598aaa1e105ac9, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/eb867314c5d34a87a49ebbd599774dc1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/e8d8c03626be40e49f5981b1c861ddc1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/596328aaad9d4e4fbd3cfaa585be273d] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp, totalSize=70.3 K 2024-11-26T16:43:19,490 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5f112ddfe974376ba0b782d7f224a26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732639389906 2024-11-26T16:43:19,491 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0522e8a2c6294d84a0cd3dc5d1e401e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732639389929 2024-11-26T16:43:19,491 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4497716c9b9e47dc95598aaa1e105ac9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732639391081 2024-11-26T16:43:19,491 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb867314c5d34a87a49ebbd599774dc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1732639392234 2024-11-26T16:43:19,491 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8d8c03626be40e49f5981b1c861ddc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732639394412 2024-11-26T16:43:19,492 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 596328aaad9d4e4fbd3cfaa585be273d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732639396626 2024-11-26T16:43:19,501 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c551e376aede51019d0df72805dd3c05#C#compaction#470 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:43:19,502 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/2c6a4a3a97ca4992a4c348de82a7f92b is 50, key is test_row_0/C:col10/1732639397773/Put/seqid=0 2024-11-26T16:43:19,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742378_1554 (size=12207) 2024-11-26T16:43:19,509 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/2c6a4a3a97ca4992a4c348de82a7f92b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/2c6a4a3a97ca4992a4c348de82a7f92b 2024-11-26T16:43:19,514 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in c551e376aede51019d0df72805dd3c05/C of c551e376aede51019d0df72805dd3c05 into 2c6a4a3a97ca4992a4c348de82a7f92b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:43:19,514 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:19,514 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., storeName=c551e376aede51019d0df72805dd3c05/C, priority=10, startTime=1732639399441; duration=0sec 2024-11-26T16:43:19,514 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:19,514 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c551e376aede51019d0df72805dd3c05:C 2024-11-26T16:43:19,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-26T16:43:19,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:19,576 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing c551e376aede51019d0df72805dd3c05 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T16:43:19,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=A 2024-11-26T16:43:19,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:19,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=B 2024-11-26T16:43:19,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:19,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=C 2024-11-26T16:43:19,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:19,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112689e7a4bcbfd1487e8983b7112e5590b7_c551e376aede51019d0df72805dd3c05 is 50, key is test_row_0/A:col10/1732639397790/Put/seqid=0 2024-11-26T16:43:19,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742379_1555 (size=12154) 2024-11-26T16:43:19,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T16:43:19,873 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/c26fd24209114532bfaa675cae280fcd as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/c26fd24209114532bfaa675cae280fcd 2024-11-26T16:43:19,879 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in c551e376aede51019d0df72805dd3c05/B of c551e376aede51019d0df72805dd3c05 into c26fd24209114532bfaa675cae280fcd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:43:19,879 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:19,879 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., storeName=c551e376aede51019d0df72805dd3c05/B, priority=10, startTime=1732639399441; duration=0sec 2024-11-26T16:43:19,879 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:19,879 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c551e376aede51019d0df72805dd3c05:B 2024-11-26T16:43:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:19,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:19,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639459970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:19,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:19,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:19,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:19,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639459970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:19,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:19,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:19,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:19,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:19,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639459971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:19,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:19,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732639399961, max timestamp=1732639399976Num uniques 859; 2024-11-26T16:43:19,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732639399961, max timestamp=1732639399976Num uniques 859; 2024-11-26T16:43:19,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:43:19,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:43:19,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732639399961, max timestamp=1732639399976Num uniques 859; 2024-11-26T16:43:19,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:43:19,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:19,992 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112689e7a4bcbfd1487e8983b7112e5590b7_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112689e7a4bcbfd1487e8983b7112e5590b7_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:19,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/c85437f1f7584ee79198fc2b6210da41, store: [table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:19,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/c85437f1f7584ee79198fc2b6210da41 is 175, key is test_row_0/A:col10/1732639397790/Put/seqid=0 2024-11-26T16:43:19,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742380_1556 (size=30955) 2024-11-26T16:43:19,998 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/c85437f1f7584ee79198fc2b6210da41 2024-11-26T16:43:20,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/17a6618bd227489fbcd9a5bcafcf1de0 is 50, key is test_row_0/B:col10/1732639397790/Put/seqid=0 2024-11-26T16:43:20,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742381_1557 (size=12001) 2024-11-26T16:43:20,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:20,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639460017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:20,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:20,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:20,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639460023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:20,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:20,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639460081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:20,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639460081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:20,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639460082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:20,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:20,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639460131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:20,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:20,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639460131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:20,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:20,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:20,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:20,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639460291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:20,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639460291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:20,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639460291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:20,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639460341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:20,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:20,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639460341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:20,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:20,408 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/17a6618bd227489fbcd9a5bcafcf1de0 2024-11-26T16:43:20,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/9450d6f0adf64efeac2f6e929303e425 is 50, key is test_row_0/C:col10/1732639397790/Put/seqid=0 2024-11-26T16:43:20,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742382_1558 (size=12001) 2024-11-26T16:43:20,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639460600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:20,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639460601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639460601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:20,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T16:43:20,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:20,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639460651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:20,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639460651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:20,819 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/9450d6f0adf64efeac2f6e929303e425 2024-11-26T16:43:20,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/c85437f1f7584ee79198fc2b6210da41 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/c85437f1f7584ee79198fc2b6210da41 2024-11-26T16:43:20,827 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/c85437f1f7584ee79198fc2b6210da41, entries=150, sequenceid=115, filesize=30.2 K 2024-11-26T16:43:20,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/17a6618bd227489fbcd9a5bcafcf1de0 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/17a6618bd227489fbcd9a5bcafcf1de0 2024-11-26T16:43:20,831 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/17a6618bd227489fbcd9a5bcafcf1de0, entries=150, sequenceid=115, filesize=11.7 K 2024-11-26T16:43:20,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/9450d6f0adf64efeac2f6e929303e425 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/9450d6f0adf64efeac2f6e929303e425 2024-11-26T16:43:20,835 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/9450d6f0adf64efeac2f6e929303e425, entries=150, sequenceid=115, filesize=11.7 K 2024-11-26T16:43:20,835 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for c551e376aede51019d0df72805dd3c05 in 1259ms, sequenceid=115, compaction requested=false 2024-11-26T16:43:20,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:20,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-26T16:43:20,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-26T16:43:20,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-26T16:43:20,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3330 sec 2024-11-26T16:43:20,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.3370 sec 2024-11-26T16:43:21,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:21,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c551e376aede51019d0df72805dd3c05 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-26T16:43:21,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=A 2024-11-26T16:43:21,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:43:21,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=B 2024-11-26T16:43:21,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:43:21,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=C 2024-11-26T16:43:21,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:43:21,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126692cebc0bd174aa6bc319a0cc2d4b8a8_c551e376aede51019d0df72805dd3c05 is 50, key is test_row_0/A:col10/1732639401116/Put/seqid=0 2024-11-26T16:43:21,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742383_1559 (size=17284) 2024-11-26T16:43:21,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639461149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:21,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:21,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639461154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:21,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:21,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639461155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:21,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:21,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:21,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:21,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639461161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:21,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:21,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639461161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:21,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:21,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:21,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639461261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:21,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:21,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639461271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:21,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:21,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639461271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:21,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639461471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:21,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:21,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639461481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:21,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639461481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:21,552 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:21,556 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126692cebc0bd174aa6bc319a0cc2d4b8a8_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126692cebc0bd174aa6bc319a0cc2d4b8a8_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:21,557 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/f34518f56643454fa232325c3a084722, store: [table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:21,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/f34518f56643454fa232325c3a084722 is 175, key is test_row_0/A:col10/1732639401116/Put/seqid=0 2024-11-26T16:43:21,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742384_1560 (size=48389) 2024-11-26T16:43:21,561 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/f34518f56643454fa232325c3a084722 2024-11-26T16:43:21,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/8a53f2789e064e41868c23afd1ee3aff is 50, key is test_row_0/B:col10/1732639401116/Put/seqid=0 2024-11-26T16:43:21,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742385_1561 (size=12151) 2024-11-26T16:43:21,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639461781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:21,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639461790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:21,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639461791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:21,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/8a53f2789e064e41868c23afd1ee3aff 2024-11-26T16:43:21,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/30e4c913a7aa4700a339c36d9e0e3fbe is 50, key is test_row_0/C:col10/1732639401116/Put/seqid=0 2024-11-26T16:43:21,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742386_1562 (size=12151) 2024-11-26T16:43:22,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639462181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:22,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639462181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:22,185 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in c551e376aede51019d0df72805dd3c05 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:43:22,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:22,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:22,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:22,185 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_0' in c551e376aede51019d0df72805dd3c05 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:43:22,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:22,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:22,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:22,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639462291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:22,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639462301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:22,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639462301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:22,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/30e4c913a7aa4700a339c36d9e0e3fbe 2024-11-26T16:43:22,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/f34518f56643454fa232325c3a084722 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/f34518f56643454fa232325c3a084722 2024-11-26T16:43:22,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/f34518f56643454fa232325c3a084722, entries=250, sequenceid=153, filesize=47.3 K 2024-11-26T16:43:22,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/8a53f2789e064e41868c23afd1ee3aff as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/8a53f2789e064e41868c23afd1ee3aff 2024-11-26T16:43:22,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/8a53f2789e064e41868c23afd1ee3aff, entries=150, sequenceid=153, filesize=11.9 K 2024-11-26T16:43:22,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/30e4c913a7aa4700a339c36d9e0e3fbe as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/30e4c913a7aa4700a339c36d9e0e3fbe 2024-11-26T16:43:22,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/30e4c913a7aa4700a339c36d9e0e3fbe, entries=150, sequenceid=153, filesize=11.9 K 2024-11-26T16:43:22,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for c551e376aede51019d0df72805dd3c05 in 1273ms, sequenceid=153, compaction requested=true 2024-11-26T16:43:22,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:22,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c551e376aede51019d0df72805dd3c05:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:43:22,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:22,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c551e376aede51019d0df72805dd3c05:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:43:22,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:22,399 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:43:22,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c551e376aede51019d0df72805dd3c05:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:43:22,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:43:22,399 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:43:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,399 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:43:22,399 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:43:22,400 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): c551e376aede51019d0df72805dd3c05/A is initiating minor compaction (all files) 2024-11-26T16:43:22,400 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): c551e376aede51019d0df72805dd3c05/B is initiating minor compaction (all files) 2024-11-26T16:43:22,400 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c551e376aede51019d0df72805dd3c05/B in TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:22,400 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c551e376aede51019d0df72805dd3c05/A in TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:22,400 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/c26fd24209114532bfaa675cae280fcd, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/17a6618bd227489fbcd9a5bcafcf1de0, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/8a53f2789e064e41868c23afd1ee3aff] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp, totalSize=35.5 K 2024-11-26T16:43:22,400 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/b8db8196b91943eba44ccf0cf73914fc, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/c85437f1f7584ee79198fc2b6210da41, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/f34518f56643454fa232325c3a084722] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp, totalSize=107.9 K 2024-11-26T16:43:22,400 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:22,400 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. files: [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/b8db8196b91943eba44ccf0cf73914fc, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/c85437f1f7584ee79198fc2b6210da41, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/f34518f56643454fa232325c3a084722] 2024-11-26T16:43:22,400 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8db8196b91943eba44ccf0cf73914fc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732639396626 2024-11-26T16:43:22,400 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting c26fd24209114532bfaa675cae280fcd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732639396626 2024-11-26T16:43:22,401 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting c85437f1f7584ee79198fc2b6210da41, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732639397784 2024-11-26T16:43:22,401 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 17a6618bd227489fbcd9a5bcafcf1de0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732639397784 2024-11-26T16:43:22,401 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting f34518f56643454fa232325c3a084722, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732639400008 2024-11-26T16:43:22,401 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a53f2789e064e41868c23afd1ee3aff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732639400008 2024-11-26T16:43:22,407 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,409 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c551e376aede51019d0df72805dd3c05#B#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:43:22,410 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/d4b31e0c3da145de9c0236437193a7aa is 50, key is test_row_0/B:col10/1732639401116/Put/seqid=0 2024-11-26T16:43:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,411 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112637c20a5fd3e14578bf8d4e9de227f5e5_c551e376aede51019d0df72805dd3c05 store=[table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,413 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112637c20a5fd3e14578bf8d4e9de227f5e5_c551e376aede51019d0df72805dd3c05, store=[table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:22,413 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112637c20a5fd3e14578bf8d4e9de227f5e5_c551e376aede51019d0df72805dd3c05 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742388_1564 (size=4469) 2024-11-26T16:43:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742387_1563 (size=12459) 2024-11-26T16:43:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-26T16:43:22,672 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:43:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,673 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:43:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-26T16:43:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T16:43:22,675 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:43:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,676 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:43:22,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:43:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T16:43:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-26T16:43:22,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:22,828 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing c551e376aede51019d0df72805dd3c05 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-26T16:43:22,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=A 2024-11-26T16:43:22,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:22,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=B 2024-11-26T16:43:22,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:22,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=C 2024-11-26T16:43:22,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:22,830 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c551e376aede51019d0df72805dd3c05#A#compaction#478 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:43:22,831 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/5aaf4885ff4744968bd9da8265821ff4 is 175, key is test_row_0/A:col10/1732639401116/Put/seqid=0 2024-11-26T16:43:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267d9015810014429dbb73774dec980c6c_c551e376aede51019d0df72805dd3c05 is 50, key is test_row_0/A:col10/1732639401153/Put/seqid=0 2024-11-26T16:43:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,837 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/d4b31e0c3da145de9c0236437193a7aa as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/d4b31e0c3da145de9c0236437193a7aa 2024-11-26T16:43:22,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742389_1565 (size=31413) 2024-11-26T16:43:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,842 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c551e376aede51019d0df72805dd3c05/B of c551e376aede51019d0df72805dd3c05 into d4b31e0c3da145de9c0236437193a7aa(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:43:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,842 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:22,842 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., storeName=c551e376aede51019d0df72805dd3c05/B, priority=13, startTime=1732639402399; duration=0sec 2024-11-26T16:43:22,842 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:43:22,842 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c551e376aede51019d0df72805dd3c05:B 2024-11-26T16:43:22,842 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:43:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,844 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:43:22,845 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): c551e376aede51019d0df72805dd3c05/C is initiating minor compaction (all files) 2024-11-26T16:43:22,845 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c551e376aede51019d0df72805dd3c05/C in TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:22,845 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/2c6a4a3a97ca4992a4c348de82a7f92b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/9450d6f0adf64efeac2f6e929303e425, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/30e4c913a7aa4700a339c36d9e0e3fbe] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp, totalSize=35.5 K 2024-11-26T16:43:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,845 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c6a4a3a97ca4992a4c348de82a7f92b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732639396626 2024-11-26T16:43:22,845 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 9450d6f0adf64efeac2f6e929303e425, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732639397784 2024-11-26T16:43:22,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742390_1566 (size=9814) 2024-11-26T16:43:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,846 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 30e4c913a7aa4700a339c36d9e0e3fbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732639400008 2024-11-26T16:43:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,854 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c551e376aede51019d0df72805dd3c05#C#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:43:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,855 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/d162e1bb84534cc88ba44aaeb12341eb is 50, key is test_row_0/C:col10/1732639401116/Put/seqid=0 2024-11-26T16:43:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742391_1567 (size=12459) 2024-11-26T16:43:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,880 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/d162e1bb84534cc88ba44aaeb12341eb as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/d162e1bb84534cc88ba44aaeb12341eb 2024-11-26T16:43:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,887 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c551e376aede51019d0df72805dd3c05/C of c551e376aede51019d0df72805dd3c05 into d162e1bb84534cc88ba44aaeb12341eb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:43:22,887 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:22,887 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., storeName=c551e376aede51019d0df72805dd3c05/C, priority=13, startTime=1732639402399; duration=0sec 2024-11-26T16:43:22,887 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:22,887 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c551e376aede51019d0df72805dd3c05:C 2024-11-26T16:43:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T16:43:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,242 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/5aaf4885ff4744968bd9da8265821ff4 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/5aaf4885ff4744968bd9da8265821ff4 2024-11-26T16:43:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,246 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c551e376aede51019d0df72805dd3c05/A of c551e376aede51019d0df72805dd3c05 into 5aaf4885ff4744968bd9da8265821ff4(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:43:23,246 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:23,246 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., storeName=c551e376aede51019d0df72805dd3c05/A, priority=13, startTime=1732639402399; duration=0sec 2024-11-26T16:43:23,246 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:23,246 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c551e376aede51019d0df72805dd3c05:A 2024-11-26T16:43:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,249 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411267d9015810014429dbb73774dec980c6c_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267d9015810014429dbb73774dec980c6c_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/fa8a825b2c5e4d38bc5dbb52cfe7551e, store: [table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:23,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/fa8a825b2c5e4d38bc5dbb52cfe7551e is 175, key is test_row_0/A:col10/1732639401153/Put/seqid=0 2024-11-26T16:43:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742392_1568 (size=22461) 2024-11-26T16:43:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,255 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=162, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/fa8a825b2c5e4d38bc5dbb52cfe7551e 2024-11-26T16:43:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/1abebbaea2a74dfca937d929015e1130 is 50, key is test_row_0/B:col10/1732639401153/Put/seqid=0 2024-11-26T16:43:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742393_1569 (size=9757) 2024-11-26T16:43:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T16:43:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:23,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639403331, max timestamp=1732639403347Num uniques 859; 2024-11-26T16:43:23,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:43:23,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639403331, max timestamp=1732639403347Num uniques 859; 2024-11-26T16:43:23,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:43:23,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732639403331, max timestamp=1732639403347Num uniques 859; 2024-11-26T16:43:23,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-26T16:43:23,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732639403374, max timestamp=1732639403413Num uniques 1018; 2024-11-26T16:43:23,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T16:43:23,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732639403374, max timestamp=1732639403413Num uniques 1018; 2024-11-26T16:43:23,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T16:43:23,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732639403374, max timestamp=1732639403413Num uniques 1018; 2024-11-26T16:43:23,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-26T16:43:23,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639463447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:23,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639463452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:23,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639463453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:23,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:23,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639463561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639463561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:23,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639463571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:23,671 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/1abebbaea2a74dfca937d929015e1130 2024-11-26T16:43:23,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/f183d2ff80df451ab0ee0b9212c99f75 is 50, key is test_row_0/C:col10/1732639401153/Put/seqid=0 2024-11-26T16:43:23,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742394_1570 (size=9757) 2024-11-26T16:43:23,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639463771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:23,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639463771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:23,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639463781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:23,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T16:43:24,083 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/f183d2ff80df451ab0ee0b9212c99f75 2024-11-26T16:43:24,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639464081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639464082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:24,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/fa8a825b2c5e4d38bc5dbb52cfe7551e as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/fa8a825b2c5e4d38bc5dbb52cfe7551e 2024-11-26T16:43:24,090 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/fa8a825b2c5e4d38bc5dbb52cfe7551e, entries=100, sequenceid=162, filesize=21.9 K 2024-11-26T16:43:24,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/1abebbaea2a74dfca937d929015e1130 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/1abebbaea2a74dfca937d929015e1130 2024-11-26T16:43:24,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639464091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:24,095 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/1abebbaea2a74dfca937d929015e1130, entries=100, sequenceid=162, filesize=9.5 K 2024-11-26T16:43:24,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/f183d2ff80df451ab0ee0b9212c99f75 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/f183d2ff80df451ab0ee0b9212c99f75 2024-11-26T16:43:24,100 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/f183d2ff80df451ab0ee0b9212c99f75, entries=100, sequenceid=162, filesize=9.5 K 2024-11-26T16:43:24,101 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for c551e376aede51019d0df72805dd3c05 in 1273ms, sequenceid=162, compaction requested=false 2024-11-26T16:43:24,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:24,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:24,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-26T16:43:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-26T16:43:24,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-26T16:43:24,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4260 sec 2024-11-26T16:43:24,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.4320 sec 2024-11-26T16:43:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:24,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c551e376aede51019d0df72805dd3c05 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-11-26T16:43:24,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=A 2024-11-26T16:43:24,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:43:24,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=B 2024-11-26T16:43:24,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:43:24,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=C 2024-11-26T16:43:24,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-26T16:43:24,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261c80ef9d13824c2b9e5a562f52964cae_c551e376aede51019d0df72805dd3c05 is 50, key is test_row_0/A:col10/1732639404204/Put/seqid=0 2024-11-26T16:43:24,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742395_1571 (size=12304) 2024-11-26T16:43:24,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639464221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:24,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639464223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:24,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:24,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639464331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:24,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639464331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:24,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:24,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639464541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:24,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:24,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639464541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:24,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:24,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:24,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639464591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:24,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639464591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:24,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639464601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:24,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:24,620 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:24,624 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261c80ef9d13824c2b9e5a562f52964cae_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261c80ef9d13824c2b9e5a562f52964cae_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:24,624 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/35710bbeb6fe459f8dabc2b26c45cb6d, store: [table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:24,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/35710bbeb6fe459f8dabc2b26c45cb6d is 175, key is test_row_0/A:col10/1732639404204/Put/seqid=0 2024-11-26T16:43:24,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742396_1572 (size=31105) 2024-11-26T16:43:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-26T16:43:24,821 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:43:24,821 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:43:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-26T16:43:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T16:43:24,823 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:43:24,824 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:43:24,824 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:43:24,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639464851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:24,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639464851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:24,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:24,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:24,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T16:43:24,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-26T16:43:24,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:24,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:24,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:24,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:24,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:25,029 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/35710bbeb6fe459f8dabc2b26c45cb6d 2024-11-26T16:43:25,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/e690038834cc4fe6b63bf2b78489d8e6 is 50, key is test_row_0/B:col10/1732639404204/Put/seqid=0 2024-11-26T16:43:25,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742397_1573 (size=12151) 2024-11-26T16:43:25,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/e690038834cc4fe6b63bf2b78489d8e6 2024-11-26T16:43:25,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/109bb75a167045b2b11648ff112b2e8c is 50, key is test_row_0/C:col10/1732639404204/Put/seqid=0 2024-11-26T16:43:25,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742398_1574 (size=12151) 2024-11-26T16:43:25,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-26T16:43:25,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:25,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:25,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:25,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:25,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:25,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T16:43:25,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-26T16:43:25,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:25,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:25,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:25,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:25,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:25,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:25,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639465361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:25,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:25,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639465361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:25,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-26T16:43:25,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:25,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:25,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:25,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:25,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:25,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T16:43:25,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/109bb75a167045b2b11648ff112b2e8c 2024-11-26T16:43:25,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/35710bbeb6fe459f8dabc2b26c45cb6d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/35710bbeb6fe459f8dabc2b26c45cb6d 2024-11-26T16:43:25,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/35710bbeb6fe459f8dabc2b26c45cb6d, entries=150, sequenceid=220, filesize=30.4 K 2024-11-26T16:43:25,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/e690038834cc4fe6b63bf2b78489d8e6 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/e690038834cc4fe6b63bf2b78489d8e6 2024-11-26T16:43:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/e690038834cc4fe6b63bf2b78489d8e6, entries=150, sequenceid=220, filesize=11.9 K 2024-11-26T16:43:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/109bb75a167045b2b11648ff112b2e8c as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/109bb75a167045b2b11648ff112b2e8c 2024-11-26T16:43:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/109bb75a167045b2b11648ff112b2e8c, entries=150, sequenceid=220, filesize=11.9 K 2024-11-26T16:43:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for c551e376aede51019d0df72805dd3c05 in 1266ms, sequenceid=220, compaction requested=true 2024-11-26T16:43:25,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:25,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c551e376aede51019d0df72805dd3c05:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:43:25,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:25,471 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:43:25,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c551e376aede51019d0df72805dd3c05:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:43:25,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:25,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c551e376aede51019d0df72805dd3c05:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:43:25,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:43:25,471 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:43:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,472 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84979 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:43:25,472 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): c551e376aede51019d0df72805dd3c05/A is initiating minor compaction (all files) 2024-11-26T16:43:25,472 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c551e376aede51019d0df72805dd3c05/A in TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,472 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/5aaf4885ff4744968bd9da8265821ff4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/fa8a825b2c5e4d38bc5dbb52cfe7551e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/35710bbeb6fe459f8dabc2b26c45cb6d] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp, totalSize=83.0 K 2024-11-26T16:43:25,472 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:25,472 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. files: [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/5aaf4885ff4744968bd9da8265821ff4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/fa8a825b2c5e4d38bc5dbb52cfe7551e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/35710bbeb6fe459f8dabc2b26c45cb6d] 2024-11-26T16:43:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,473 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:43:25,473 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): c551e376aede51019d0df72805dd3c05/B is initiating minor compaction (all files) 2024-11-26T16:43:25,473 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c551e376aede51019d0df72805dd3c05/B in TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,473 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/d4b31e0c3da145de9c0236437193a7aa, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/1abebbaea2a74dfca937d929015e1130, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/e690038834cc4fe6b63bf2b78489d8e6] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp, totalSize=33.6 K 2024-11-26T16:43:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,473 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5aaf4885ff4744968bd9da8265821ff4, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732639400008 2024-11-26T16:43:25,473 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting d4b31e0c3da145de9c0236437193a7aa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732639400008 2024-11-26T16:43:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,474 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa8a825b2c5e4d38bc5dbb52cfe7551e, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732639401153 2024-11-26T16:43:25,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,474 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 1abebbaea2a74dfca937d929015e1130, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732639401153 2024-11-26T16:43:25,474 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35710bbeb6fe459f8dabc2b26c45cb6d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732639403444 2024-11-26T16:43:25,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,474 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting e690038834cc4fe6b63bf2b78489d8e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732639403444 2024-11-26T16:43:25,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,479 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:25,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,481 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c551e376aede51019d0df72805dd3c05#B#compaction#487 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:43:25,482 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411261fc36078fc544b3e9f87b533ef95e391_c551e376aede51019d0df72805dd3c05 store=[table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:25,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,482 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/5152b5535ea54a63838c516529ad5c21 is 50, key is test_row_0/B:col10/1732639404204/Put/seqid=0 2024-11-26T16:43:25,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,483 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411261fc36078fc544b3e9f87b533ef95e391_c551e376aede51019d0df72805dd3c05, store=[table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:25,483 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411261fc36078fc544b3e9f87b533ef95e391_c551e376aede51019d0df72805dd3c05 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742399_1575 (size=12561) 2024-11-26T16:43:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742400_1576 (size=4469) 2024-11-26T16:43:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,489 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c551e376aede51019d0df72805dd3c05#A#compaction#486 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:43:25,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,489 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/3aa52fb94c094aeda4f2f709d42ace4f is 175, key is test_row_0/A:col10/1732639404204/Put/seqid=0 2024-11-26T16:43:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,493 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/5152b5535ea54a63838c516529ad5c21 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/5152b5535ea54a63838c516529ad5c21 2024-11-26T16:43:25,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742401_1577 (size=31515) 2024-11-26T16:43:25,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,501 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c551e376aede51019d0df72805dd3c05/B of c551e376aede51019d0df72805dd3c05 into 5152b5535ea54a63838c516529ad5c21(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:43:25,501 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:25,501 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., storeName=c551e376aede51019d0df72805dd3c05/B, priority=13, startTime=1732639405471; duration=0sec 2024-11-26T16:43:25,501 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:43:25,501 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c551e376aede51019d0df72805dd3c05:B 2024-11-26T16:43:25,501 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:43:25,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,502 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:43:25,503 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): c551e376aede51019d0df72805dd3c05/C is initiating minor compaction (all files) 2024-11-26T16:43:25,503 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c551e376aede51019d0df72805dd3c05/C in TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:25,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,503 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/d162e1bb84534cc88ba44aaeb12341eb, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/f183d2ff80df451ab0ee0b9212c99f75, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/109bb75a167045b2b11648ff112b2e8c] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp, totalSize=33.6 K 2024-11-26T16:43:25,503 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting d162e1bb84534cc88ba44aaeb12341eb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732639400008 2024-11-26T16:43:25,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,503 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting f183d2ff80df451ab0ee0b9212c99f75, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732639401153 2024-11-26T16:43:25,504 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 109bb75a167045b2b11648ff112b2e8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732639403444 2024-11-26T16:43:25,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,512 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c551e376aede51019d0df72805dd3c05#C#compaction#488 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:43:25,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,512 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/2297e39bf135446b974641c724f132d9 is 50, key is test_row_0/C:col10/1732639404204/Put/seqid=0 2024-11-26T16:43:25,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742402_1578 (size=12561) 2024-11-26T16:43:25,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,524 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/2297e39bf135446b974641c724f132d9 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/2297e39bf135446b974641c724f132d9 2024-11-26T16:43:25,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,528 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c551e376aede51019d0df72805dd3c05/C of c551e376aede51019d0df72805dd3c05 into 2297e39bf135446b974641c724f132d9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:43:25,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,528 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:25,528 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., storeName=c551e376aede51019d0df72805dd3c05/C, priority=13, startTime=1732639405471; duration=0sec 2024-11-26T16:43:25,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,528 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:25,528 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c551e376aede51019d0df72805dd3c05:C 2024-11-26T16:43:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-26T16:43:25,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,588 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing c551e376aede51019d0df72805dd3c05 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-26T16:43:25,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=A 2024-11-26T16:43:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:25,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=B 2024-11-26T16:43:25,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:25,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=C 2024-11-26T16:43:25,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b6b9bab2ab2640049a8c56235e8be886_c551e376aede51019d0df72805dd3c05 is 50, key is test_row_0/A:col10/1732639404219/Put/seqid=0 2024-11-26T16:43:25,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742403_1579 (size=9814) 2024-11-26T16:43:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,603 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b6b9bab2ab2640049a8c56235e8be886_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b6b9bab2ab2640049a8c56235e8be886_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/dfb0757af1c349cf80777e5a21b7cc60, store: [table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/dfb0757af1c349cf80777e5a21b7cc60 is 175, key is test_row_0/A:col10/1732639404219/Put/seqid=0 2024-11-26T16:43:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:25,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742404_1580 (size=22461) 2024-11-26T16:43:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:25,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732639405641, max timestamp=1732639405648Num uniques 859; 2024-11-26T16:43:25,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:43:25,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732639405641, max timestamp=1732639405648Num uniques 859; 2024-11-26T16:43:25,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:43:25,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732639405641, max timestamp=1732639405648Num uniques 859; 2024-11-26T16:43:25,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-26T16:43:25,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639465681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:25,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:25,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639465682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:25,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:25,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:25,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732639405659, max timestamp=1732639405686Num uniques 1068; 2024-11-26T16:43:25,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-26T16:43:25,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732639405659, max timestamp=1732639405686Num uniques 1068; 2024-11-26T16:43:25,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-26T16:43:25,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732639405659, max timestamp=1732639405686Num uniques 1068; 2024-11-26T16:43:25,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-26T16:43:25,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732639405790, max timestamp=1732639405805Num uniques 1027; 2024-11-26T16:43:25,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-26T16:43:25,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732639405790, max timestamp=1732639405805Num uniques 1027; 2024-11-26T16:43:25,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732639405790, max timestamp=1732639405805Num uniques 1027; 2024-11-26T16:43:25,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-26T16:43:25,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-26T16:43:25,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732639405833, max timestamp=1732639405852Num uniques 1086; 2024-11-26T16:43:25,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732639405833, max timestamp=1732639405852Num uniques 1086; 2024-11-26T16:43:25,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-26T16:43:25,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-26T16:43:25,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732639405833, max timestamp=1732639405852Num uniques 1086; 2024-11-26T16:43:25,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-26T16:43:25,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732639405881, max timestamp=1732639405897Num uniques 1095; 2024-11-26T16:43:25,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-26T16:43:25,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732639405887, max timestamp=1732639405897Num uniques 1095; 2024-11-26T16:43:25,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-26T16:43:25,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732639405887, max timestamp=1732639405897Num uniques 1095; 2024-11-26T16:43:25,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-26T16:43:25,907 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/3aa52fb94c094aeda4f2f709d42ace4f as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/3aa52fb94c094aeda4f2f709d42ace4f 2024-11-26T16:43:25,913 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c551e376aede51019d0df72805dd3c05/A of c551e376aede51019d0df72805dd3c05 into 3aa52fb94c094aeda4f2f709d42ace4f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:43:25,913 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:25,913 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., storeName=c551e376aede51019d0df72805dd3c05/A, priority=13, startTime=1732639405471; duration=0sec 2024-11-26T16:43:25,913 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:25,913 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c551e376aede51019d0df72805dd3c05:A 2024-11-26T16:43:25,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732639405938, max timestamp=1732639405951Num uniques 1154; 2024-11-26T16:43:25,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732639405938, max timestamp=1732639405951Num uniques 1054; 2024-11-26T16:43:25,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-11-26T16:43:25,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-26T16:43:25,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732639405938, max timestamp=1732639405951Num uniques 1054; 2024-11-26T16:43:25,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-26T16:43:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T16:43:25,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639465992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:25,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732639405986, max timestamp=1732639405994Num uniques 1063; 2024-11-26T16:43:25,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-26T16:43:25,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732639405986, max timestamp=1732639405994Num uniques 1063; 2024-11-26T16:43:25,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-26T16:43:25,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732639405986, max timestamp=1732639405994Num uniques 1063; 2024-11-26T16:43:25,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-26T16:43:26,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:26,022 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/dfb0757af1c349cf80777e5a21b7cc60 2024-11-26T16:43:26,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/b9cbf75e6c0c428faba33d37aaac115d is 50, key is test_row_0/B:col10/1732639404219/Put/seqid=0 2024-11-26T16:43:26,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742405_1581 (size=9757) 2024-11-26T16:43:26,034 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/b9cbf75e6c0c428faba33d37aaac115d 2024-11-26T16:43:26,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/e4cabe134c444ed8a527041514ad9e2f is 50, key is test_row_0/C:col10/1732639404219/Put/seqid=0 2024-11-26T16:43:26,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742406_1582 (size=9757) 2024-11-26T16:43:26,044 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/e4cabe134c444ed8a527041514ad9e2f 2024-11-26T16:43:26,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639466044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:26,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:26,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732639406018, max timestamp=1732639406046Num uniques 1072; 2024-11-26T16:43:26,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-26T16:43:26,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732639406018, max timestamp=1732639406046Num uniques 1072; 2024-11-26T16:43:26,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732639406018, max timestamp=1732639406046Num uniques 1072; 2024-11-26T16:43:26,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-26T16:43:26,051 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2dedfe7466e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-26T16:43:26,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/dfb0757af1c349cf80777e5a21b7cc60 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/dfb0757af1c349cf80777e5a21b7cc60 2024-11-26T16:43:26,060 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/dfb0757af1c349cf80777e5a21b7cc60, entries=100, sequenceid=230, filesize=21.9 K 2024-11-26T16:43:26,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/b9cbf75e6c0c428faba33d37aaac115d as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/b9cbf75e6c0c428faba33d37aaac115d 2024-11-26T16:43:26,068 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/b9cbf75e6c0c428faba33d37aaac115d, entries=100, sequenceid=230, filesize=9.5 K 2024-11-26T16:43:26,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/e4cabe134c444ed8a527041514ad9e2f as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/e4cabe134c444ed8a527041514ad9e2f 2024-11-26T16:43:26,073 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/e4cabe134c444ed8a527041514ad9e2f, entries=100, sequenceid=230, filesize=9.5 K 2024-11-26T16:43:26,074 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=60.38 KB/61830 for c551e376aede51019d0df72805dd3c05 in 485ms, sequenceid=230, compaction requested=false 2024-11-26T16:43:26,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:26,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:26,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-26T16:43:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-26T16:43:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:26,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c551e376aede51019d0df72805dd3c05 3/3 column families, dataSize=67.09 KB heapSize=184.55 KB 2024-11-26T16:43:26,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=A 2024-11-26T16:43:26,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:43:26,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=B 2024-11-26T16:43:26,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:43:26,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=C 2024-11-26T16:43:26,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:43:26,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-26T16:43:26,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2520 sec 2024-11-26T16:43:26,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.2580 sec 2024-11-26T16:43:26,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126627231c7997e4cd6ba4613f06e588529_c551e376aede51019d0df72805dd3c05 is 50, key is test_row_0/A:col10/1732639406075/Put/seqid=0 2024-11-26T16:43:26,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742407_1583 (size=14994) 2024-11-26T16:43:26,095 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:26,098 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126627231c7997e4cd6ba4613f06e588529_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126627231c7997e4cd6ba4613f06e588529_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:26,098 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/ead513f24897446bb9dd145744f94c41, store: [table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:26,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/ead513f24897446bb9dd145744f94c41 is 175, key is test_row_0/A:col10/1732639406075/Put/seqid=0 2024-11-26T16:43:26,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742408_1584 (size=39949) 2024-11-26T16:43:26,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639466149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:26,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:26,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639466151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:26,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:26,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639466155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:26,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:26,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639466260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:26,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639466271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,276 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:26,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639466360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:26,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639466371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:26,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639466371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:26,377 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_0' in c551e376aede51019d0df72805dd3c05 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:43:26,377 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in c551e376aede51019d0df72805dd3c05 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:43:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:26,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639466471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:26,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:26,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639466481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:26,503 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=378, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/ead513f24897446bb9dd145744f94c41 2024-11-26T16:43:26,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/9dd5271ec6a54cb481068e15c04e3705 is 50, key is test_row_0/B:col10/1732639406075/Put/seqid=0 2024-11-26T16:43:26,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742409_1585 (size=12301) 2024-11-26T16:43:26,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639466671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:26,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:26,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639466781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:26,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639466791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:26,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:26,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:26,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/9dd5271ec6a54cb481068e15c04e3705 2024-11-26T16:43:26,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/55916c67b2a74c1e934fd9ce933f5825 is 50, key is test_row_0/C:col10/1732639406075/Put/seqid=0 2024-11-26T16:43:26,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742410_1586 (size=12301) 2024-11-26T16:43:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-26T16:43:26,971 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:43:26,972 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:43:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-26T16:43:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T16:43:26,973 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:43:26,974 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:43:26,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:43:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T16:43:27,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-26T16:43:27,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:27,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:27,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:27,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:27,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:27,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:27,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639467181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:27,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:27,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:27,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:27,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-26T16:43:27,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:27,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:27,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:27,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:27,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T16:43:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T16:43:27,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639467301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:27,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639467301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:27,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:27,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:27,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:27,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:27,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:27,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/55916c67b2a74c1e934fd9ce933f5825 2024-11-26T16:43:27,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/ead513f24897446bb9dd145744f94c41 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/ead513f24897446bb9dd145744f94c41 2024-11-26T16:43:27,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/ead513f24897446bb9dd145744f94c41, entries=200, sequenceid=378, filesize=39.0 K 2024-11-26T16:43:27,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/9dd5271ec6a54cb481068e15c04e3705 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/9dd5271ec6a54cb481068e15c04e3705 2024-11-26T16:43:27,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/9dd5271ec6a54cb481068e15c04e3705, entries=150, sequenceid=378, filesize=12.0 K 2024-11-26T16:43:27,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/55916c67b2a74c1e934fd9ce933f5825 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/55916c67b2a74c1e934fd9ce933f5825 2024-11-26T16:43:27,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/55916c67b2a74c1e934fd9ce933f5825, entries=150, sequenceid=378, filesize=12.0 K 2024-11-26T16:43:27,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~184.50 KB/188928, currentSize=114.05 KB/116790 for c551e376aede51019d0df72805dd3c05 in 1263ms, sequenceid=378, compaction requested=true 2024-11-26T16:43:27,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:27,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c551e376aede51019d0df72805dd3c05:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:43:27,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:27,341 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:43:27,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c551e376aede51019d0df72805dd3c05:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:43:27,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:27,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c551e376aede51019d0df72805dd3c05:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:43:27,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:43:27,341 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:43:27,342 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:43:27,342 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:43:27,342 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): c551e376aede51019d0df72805dd3c05/B is initiating minor compaction (all files) 2024-11-26T16:43:27,342 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): c551e376aede51019d0df72805dd3c05/A is initiating minor compaction (all files) 2024-11-26T16:43:27,342 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c551e376aede51019d0df72805dd3c05/B in TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:27,342 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c551e376aede51019d0df72805dd3c05/A in TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:27,342 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/5152b5535ea54a63838c516529ad5c21, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/b9cbf75e6c0c428faba33d37aaac115d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/9dd5271ec6a54cb481068e15c04e3705] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp, totalSize=33.8 K 2024-11-26T16:43:27,342 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/3aa52fb94c094aeda4f2f709d42ace4f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/dfb0757af1c349cf80777e5a21b7cc60, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/ead513f24897446bb9dd145744f94c41] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp, totalSize=91.7 K 2024-11-26T16:43:27,342 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:27,342 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. files: [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/3aa52fb94c094aeda4f2f709d42ace4f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/dfb0757af1c349cf80777e5a21b7cc60, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/ead513f24897446bb9dd145744f94c41] 2024-11-26T16:43:27,342 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 5152b5535ea54a63838c516529ad5c21, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732639403444 2024-11-26T16:43:27,342 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aa52fb94c094aeda4f2f709d42ace4f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732639403444 2024-11-26T16:43:27,343 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting b9cbf75e6c0c428faba33d37aaac115d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732639404219 2024-11-26T16:43:27,343 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfb0757af1c349cf80777e5a21b7cc60, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732639404219 2024-11-26T16:43:27,343 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dd5271ec6a54cb481068e15c04e3705, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732639406053 2024-11-26T16:43:27,343 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting ead513f24897446bb9dd145744f94c41, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732639406053 2024-11-26T16:43:27,348 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:27,350 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c551e376aede51019d0df72805dd3c05#B#compaction#496 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:43:27,350 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241126b5c04eb2ba98407aadcd5a1598cc62ac_c551e376aede51019d0df72805dd3c05 store=[table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:27,350 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/a3de5ffbb52f4eedb2dfa6a68ec09367 is 50, key is test_row_0/B:col10/1732639406075/Put/seqid=0 2024-11-26T16:43:27,352 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241126b5c04eb2ba98407aadcd5a1598cc62ac_c551e376aede51019d0df72805dd3c05, store=[table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:27,352 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126b5c04eb2ba98407aadcd5a1598cc62ac_c551e376aede51019d0df72805dd3c05 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:27,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742411_1587 (size=12813) 2024-11-26T16:43:27,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742412_1588 (size=4469) 2024-11-26T16:43:27,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-26T16:43:27,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:27,433 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing c551e376aede51019d0df72805dd3c05 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-26T16:43:27,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=A 2024-11-26T16:43:27,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:27,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=B 2024-11-26T16:43:27,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:27,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=C 2024-11-26T16:43:27,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:27,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126cf8cee803a13437baafcb539a924d7b4_c551e376aede51019d0df72805dd3c05 is 50, key is test_row_0/A:col10/1732639406148/Put/seqid=0 2024-11-26T16:43:27,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742413_1589 (size=12454) 2024-11-26T16:43:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T16:43:27,757 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c551e376aede51019d0df72805dd3c05#A#compaction#495 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:43:27,758 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/e0d74de5c85f4a03b4d5d54ed780a22b is 175, key is test_row_0/A:col10/1732639406075/Put/seqid=0 2024-11-26T16:43:27,760 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/a3de5ffbb52f4eedb2dfa6a68ec09367 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/a3de5ffbb52f4eedb2dfa6a68ec09367 2024-11-26T16:43:27,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742414_1590 (size=31767) 2024-11-26T16:43:27,766 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c551e376aede51019d0df72805dd3c05/B of c551e376aede51019d0df72805dd3c05 into a3de5ffbb52f4eedb2dfa6a68ec09367(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:43:27,766 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:27,766 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., storeName=c551e376aede51019d0df72805dd3c05/B, priority=13, startTime=1732639407341; duration=0sec 2024-11-26T16:43:27,766 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:43:27,766 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c551e376aede51019d0df72805dd3c05:B 2024-11-26T16:43:27,766 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T16:43:27,768 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T16:43:27,768 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): c551e376aede51019d0df72805dd3c05/C is initiating minor compaction (all files) 2024-11-26T16:43:27,768 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c551e376aede51019d0df72805dd3c05/C in TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:27,768 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/2297e39bf135446b974641c724f132d9, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/e4cabe134c444ed8a527041514ad9e2f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/55916c67b2a74c1e934fd9ce933f5825] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp, totalSize=33.8 K 2024-11-26T16:43:27,768 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 2297e39bf135446b974641c724f132d9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732639403444 2024-11-26T16:43:27,769 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting e4cabe134c444ed8a527041514ad9e2f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732639404219 2024-11-26T16:43:27,769 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 55916c67b2a74c1e934fd9ce933f5825, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732639406053 2024-11-26T16:43:27,775 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c551e376aede51019d0df72805dd3c05#C#compaction#498 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:43:27,775 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/1b78d9e1349b48d5a9e355bf5e16cd8e is 50, key is test_row_0/C:col10/1732639406075/Put/seqid=0 2024-11-26T16:43:27,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742415_1591 (size=12813) 2024-11-26T16:43:27,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:27,847 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241126cf8cee803a13437baafcb539a924d7b4_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cf8cee803a13437baafcb539a924d7b4_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:27,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/700dc4d6cc124b47a7141f810cb1c4e0, store: [table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:27,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/700dc4d6cc124b47a7141f810cb1c4e0 is 175, key is test_row_0/A:col10/1732639406148/Put/seqid=0 2024-11-26T16:43:27,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742416_1592 (size=31255) 2024-11-26T16:43:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T16:43:28,166 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/e0d74de5c85f4a03b4d5d54ed780a22b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/e0d74de5c85f4a03b4d5d54ed780a22b 2024-11-26T16:43:28,171 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c551e376aede51019d0df72805dd3c05/A of c551e376aede51019d0df72805dd3c05 into e0d74de5c85f4a03b4d5d54ed780a22b(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:43:28,171 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:28,171 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., storeName=c551e376aede51019d0df72805dd3c05/A, priority=13, startTime=1732639407341; duration=0sec 2024-11-26T16:43:28,171 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:28,172 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c551e376aede51019d0df72805dd3c05:A 2024-11-26T16:43:28,190 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/1b78d9e1349b48d5a9e355bf5e16cd8e as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/1b78d9e1349b48d5a9e355bf5e16cd8e 2024-11-26T16:43:28,195 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c551e376aede51019d0df72805dd3c05/C of c551e376aede51019d0df72805dd3c05 into 1b78d9e1349b48d5a9e355bf5e16cd8e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:43:28,195 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:28,195 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., storeName=c551e376aede51019d0df72805dd3c05/C, priority=13, startTime=1732639407341; duration=0sec 2024-11-26T16:43:28,195 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:28,195 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c551e376aede51019d0df72805dd3c05:C 2024-11-26T16:43:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:28,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:28,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639468240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:28,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:28,253 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=398, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/700dc4d6cc124b47a7141f810cb1c4e0 2024-11-26T16:43:28,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/a0ca6d063d004e6985c9fe2aada0b3f5 is 50, key is test_row_0/B:col10/1732639406148/Put/seqid=0 2024-11-26T16:43:28,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742417_1593 (size=12301) 2024-11-26T16:43:28,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639468311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:28,315 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in c551e376aede51019d0df72805dd3c05 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:43:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:28,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639468320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:28,324 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in c551e376aede51019d0df72805dd3c05 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:43:28,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:28,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:28,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:28,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639468351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:28,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:28,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:28,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:28,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52964 deadline: 1732639468381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:28,384 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_2' in c551e376aede51019d0df72805dd3c05 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:43:28,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:28,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:28,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:28,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52952 deadline: 1732639468391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:28,395 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_0' in c551e376aede51019d0df72805dd3c05 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:43:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:28,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639468561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:28,669 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/a0ca6d063d004e6985c9fe2aada0b3f5 2024-11-26T16:43:28,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/bbd14811740c49308a81855b7e3a8b01 is 50, key is test_row_0/C:col10/1732639406148/Put/seqid=0 2024-11-26T16:43:28,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742418_1594 (size=12301) 2024-11-26T16:43:28,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639468871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:28,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:28,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:28,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:29,082 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/bbd14811740c49308a81855b7e3a8b01 2024-11-26T16:43:29,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/700dc4d6cc124b47a7141f810cb1c4e0 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/700dc4d6cc124b47a7141f810cb1c4e0 2024-11-26T16:43:29,089 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/700dc4d6cc124b47a7141f810cb1c4e0, entries=150, sequenceid=398, filesize=30.5 K 2024-11-26T16:43:29,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/a0ca6d063d004e6985c9fe2aada0b3f5 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/a0ca6d063d004e6985c9fe2aada0b3f5 2024-11-26T16:43:29,094 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/a0ca6d063d004e6985c9fe2aada0b3f5, entries=150, sequenceid=398, filesize=12.0 K 2024-11-26T16:43:29,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/bbd14811740c49308a81855b7e3a8b01 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/bbd14811740c49308a81855b7e3a8b01 2024-11-26T16:43:29,098 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/bbd14811740c49308a81855b7e3a8b01, entries=150, sequenceid=398, filesize=12.0 K 2024-11-26T16:43:29,099 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c551e376aede51019d0df72805dd3c05 in 1666ms, sequenceid=398, compaction requested=false 2024-11-26T16:43:29,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:29,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:29,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-26T16:43:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-26T16:43:29,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-26T16:43:29,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1270 sec 2024-11-26T16:43:29,104 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.1310 sec 2024-11-26T16:43:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-26T16:43:29,121 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:43:29,122 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-26T16:43:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-26T16:43:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T16:43:29,124 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-26T16:43:29,124 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T16:43:29,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T16:43:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T16:43:29,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38109 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-26T16:43:29,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:29,277 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing c551e376aede51019d0df72805dd3c05 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-26T16:43:29,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=A 2024-11-26T16:43:29,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:29,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=B 2024-11-26T16:43:29,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:29,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=C 2024-11-26T16:43:29,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:29,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112671127419875d41f499c94c7c30213f68_c551e376aede51019d0df72805dd3c05 is 50, key is test_row_0/A:col10/1732639408233/Put/seqid=0 2024-11-26T16:43:29,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742419_1595 (size=12454) 2024-11-26T16:43:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:29,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. as already flushing 2024-11-26T16:43:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T16:43:29,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639469468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:29,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:29,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:29,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:29,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639469581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:29,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:29,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:29,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:29,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:29,692 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112671127419875d41f499c94c7c30213f68_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112671127419875d41f499c94c7c30213f68_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:29,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/3d68f90aae8341e8a1315577d4832496, store: [table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:29,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/3d68f90aae8341e8a1315577d4832496 is 175, key is test_row_0/A:col10/1732639408233/Put/seqid=0 2024-11-26T16:43:29,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742420_1596 (size=31255) 2024-11-26T16:43:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T16:43:29,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639469791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:29,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:29,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:29,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:29,903 INFO [Thread-2363 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2363. 2024-11-26T16:43:29,904 DEBUG [Thread-2363 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:43:29,904 DEBUG [Thread-2363 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:29,904 DEBUG [Thread-2363 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:29,904 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:29,906 INFO [Thread-2359 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2359. 2024-11-26T16:43:29,906 DEBUG [Thread-2359 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:43:29,906 DEBUG [Thread-2359 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:29,906 DEBUG [Thread-2359 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:29,906 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:29,907 INFO [Thread-2365 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2365. 2024-11-26T16:43:29,907 DEBUG [Thread-2365 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:43:29,907 DEBUG [Thread-2365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:29,907 DEBUG [Thread-2365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:29,907 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:29,908 INFO [Thread-2361 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2361. 2024-11-26T16:43:29,908 DEBUG [Thread-2361 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:43:29,908 DEBUG [Thread-2361 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:29,908 DEBUG [Thread-2361 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:29,908 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:29,909 INFO [Thread-2367 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2367. 2024-11-26T16:43:29,909 DEBUG [Thread-2367 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:43:29,909 DEBUG [Thread-2367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:29,909 DEBUG [Thread-2367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:29,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:30,097 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/3d68f90aae8341e8a1315577d4832496 2024-11-26T16:43:30,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:30,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639470101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:30,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/cf03af87f080444b9cbbea94f00d4ee2 is 50, key is test_row_0/B:col10/1732639408233/Put/seqid=0 2024-11-26T16:43:30,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742421_1597 (size=12301) 2024-11-26T16:43:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T16:43:30,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52998 deadline: 1732639470331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:30,332 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in c551e376aede51019d0df72805dd3c05 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:43:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:30,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52982 deadline: 1732639470341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:30,342 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 2dedfe7466e8,38109,1732639238134 for 'test_row_1' in c551e376aede51019d0df72805dd3c05 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-26T16:43:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:30,506 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/cf03af87f080444b9cbbea94f00d4ee2 2024-11-26T16:43:30,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/437c436bf2304c4c9a9d13a531c8dfb1 is 50, key is test_row_0/C:col10/1732639408233/Put/seqid=0 2024-11-26T16:43:30,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742422_1598 (size=12301) 2024-11-26T16:43:30,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-26T16:43:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52936 deadline: 1732639470611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:30,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 , the old value is region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:30,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T16:43:30,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., hostname=2dedfe7466e8,38109,1732639238134, seqNum=5 because the exception is null or not the one we care about 2024-11-26T16:43:30,915 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/437c436bf2304c4c9a9d13a531c8dfb1 2024-11-26T16:43:30,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/3d68f90aae8341e8a1315577d4832496 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/3d68f90aae8341e8a1315577d4832496 2024-11-26T16:43:30,921 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/3d68f90aae8341e8a1315577d4832496, entries=150, sequenceid=413, filesize=30.5 K 2024-11-26T16:43:30,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/cf03af87f080444b9cbbea94f00d4ee2 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/cf03af87f080444b9cbbea94f00d4ee2 2024-11-26T16:43:30,924 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/cf03af87f080444b9cbbea94f00d4ee2, entries=150, sequenceid=413, filesize=12.0 K 2024-11-26T16:43:30,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/437c436bf2304c4c9a9d13a531c8dfb1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/437c436bf2304c4c9a9d13a531c8dfb1 2024-11-26T16:43:30,927 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/437c436bf2304c4c9a9d13a531c8dfb1, entries=150, sequenceid=413, filesize=12.0 K 2024-11-26T16:43:30,928 INFO [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c551e376aede51019d0df72805dd3c05 in 1651ms, sequenceid=413, compaction requested=true 2024-11-26T16:43:30,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:30,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:30,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2dedfe7466e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-26T16:43:30,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-26T16:43:30,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-26T16:43:30,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8050 sec 2024-11-26T16:43:30,932 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.8090 sec 2024-11-26T16:43:31,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-26T16:43:31,271 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:43:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38109 {}] regionserver.HRegion(8855): Flush requested on c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:31,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c551e376aede51019d0df72805dd3c05 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-26T16:43:31,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=A 2024-11-26T16:43:31,623 INFO [Thread-2350 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2350. 2024-11-26T16:43:31,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:43:31,623 DEBUG [Thread-2350 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:43:31,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=B 2024-11-26T16:43:31,623 DEBUG [Thread-2350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:31,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:43:31,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=C 2024-11-26T16:43:31,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-26T16:43:31,623 DEBUG [Thread-2350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:31,623 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:31,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112631f7181480264bfa94c579a2bc4d93cc_c551e376aede51019d0df72805dd3c05 is 50, key is test_row_0/A:col10/1732639409437/Put/seqid=0 2024-11-26T16:43:31,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742423_1599 (size=12454) 2024-11-26T16:43:32,032 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:32,035 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112631f7181480264bfa94c579a2bc4d93cc_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112631f7181480264bfa94c579a2bc4d93cc_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:32,035 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/3630acf95c1a4a22ae3ea0db28357a08, store: [table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:32,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/3630acf95c1a4a22ae3ea0db28357a08 is 175, key is test_row_0/A:col10/1732639409437/Put/seqid=0 2024-11-26T16:43:32,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742424_1600 (size=31255) 2024-11-26T16:43:32,423 INFO [Thread-2348 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2348. 2024-11-26T16:43:32,423 DEBUG [Thread-2348 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:43:32,423 DEBUG [Thread-2348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:32,423 DEBUG [Thread-2348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:32,423 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:32,432 INFO [Thread-2354 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2354. 2024-11-26T16:43:32,432 DEBUG [Thread-2354 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:43:32,432 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:32,432 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:32,432 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:32,440 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=434, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/3630acf95c1a4a22ae3ea0db28357a08 2024-11-26T16:43:32,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/8538e3d4ab934db9b22751bc65b7d828 is 50, key is test_row_0/B:col10/1732639409437/Put/seqid=0 2024-11-26T16:43:32,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742425_1601 (size=12301) 2024-11-26T16:43:32,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/8538e3d4ab934db9b22751bc65b7d828 2024-11-26T16:43:32,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/87defdab19934d7f838af475dd2ec4f3 is 50, key is test_row_0/C:col10/1732639409437/Put/seqid=0 2024-11-26T16:43:32,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742426_1602 (size=12301) 2024-11-26T16:43:33,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/87defdab19934d7f838af475dd2ec4f3 2024-11-26T16:43:33,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/3630acf95c1a4a22ae3ea0db28357a08 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/3630acf95c1a4a22ae3ea0db28357a08 2024-11-26T16:43:33,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/3630acf95c1a4a22ae3ea0db28357a08, entries=150, sequenceid=434, filesize=30.5 K 2024-11-26T16:43:33,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/8538e3d4ab934db9b22751bc65b7d828 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/8538e3d4ab934db9b22751bc65b7d828 2024-11-26T16:43:33,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/8538e3d4ab934db9b22751bc65b7d828, entries=150, sequenceid=434, filesize=12.0 K 2024-11-26T16:43:33,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/87defdab19934d7f838af475dd2ec4f3 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/87defdab19934d7f838af475dd2ec4f3 2024-11-26T16:43:33,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/87defdab19934d7f838af475dd2ec4f3, entries=150, sequenceid=434, filesize=12.0 K 2024-11-26T16:43:33,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=13.42 KB/13740 for c551e376aede51019d0df72805dd3c05 in 1649ms, sequenceid=434, compaction requested=true 2024-11-26T16:43:33,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:33,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c551e376aede51019d0df72805dd3c05:A, priority=-2147483648, current under compaction store size is 1 2024-11-26T16:43:33,271 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:43:33,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:33,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c551e376aede51019d0df72805dd3c05:B, priority=-2147483648, current under compaction store size is 2 2024-11-26T16:43:33,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:33,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c551e376aede51019d0df72805dd3c05:C, priority=-2147483648, current under compaction store size is 3 2024-11-26T16:43:33,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:43:33,271 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:43:33,272 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:43:33,272 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1541): c551e376aede51019d0df72805dd3c05/A is initiating minor compaction (all files) 2024-11-26T16:43:33,272 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c551e376aede51019d0df72805dd3c05/A in TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:33,272 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/e0d74de5c85f4a03b4d5d54ed780a22b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/700dc4d6cc124b47a7141f810cb1c4e0, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/3d68f90aae8341e8a1315577d4832496, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/3630acf95c1a4a22ae3ea0db28357a08] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp, totalSize=122.6 K 2024-11-26T16:43:33,272 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:43:33,272 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:33,272 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. files: [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/e0d74de5c85f4a03b4d5d54ed780a22b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/700dc4d6cc124b47a7141f810cb1c4e0, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/3d68f90aae8341e8a1315577d4832496, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/3630acf95c1a4a22ae3ea0db28357a08] 2024-11-26T16:43:33,272 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): c551e376aede51019d0df72805dd3c05/B is initiating minor compaction (all files) 2024-11-26T16:43:33,272 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c551e376aede51019d0df72805dd3c05/B in TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:33,272 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/a3de5ffbb52f4eedb2dfa6a68ec09367, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/a0ca6d063d004e6985c9fe2aada0b3f5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/cf03af87f080444b9cbbea94f00d4ee2, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/8538e3d4ab934db9b22751bc65b7d828] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp, totalSize=48.6 K 2024-11-26T16:43:33,273 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0d74de5c85f4a03b4d5d54ed780a22b, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732639406053 2024-11-26T16:43:33,273 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a3de5ffbb52f4eedb2dfa6a68ec09367, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732639406053 2024-11-26T16:43:33,273 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 700dc4d6cc124b47a7141f810cb1c4e0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732639406146 2024-11-26T16:43:33,273 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting a0ca6d063d004e6985c9fe2aada0b3f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732639406146 2024-11-26T16:43:33,273 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d68f90aae8341e8a1315577d4832496, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732639408210 2024-11-26T16:43:33,273 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting cf03af87f080444b9cbbea94f00d4ee2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732639408210 2024-11-26T16:43:33,273 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3630acf95c1a4a22ae3ea0db28357a08, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732639409437 2024-11-26T16:43:33,273 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 8538e3d4ab934db9b22751bc65b7d828, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732639409437 2024-11-26T16:43:33,279 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:33,280 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112643e52d1b3389497eaa91371947bfb352_c551e376aede51019d0df72805dd3c05 store=[table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:33,283 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c551e376aede51019d0df72805dd3c05#B#compaction#508 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:43:33,283 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/3fa0ce86883b4a39943d88afd6671cfe is 50, key is test_row_0/B:col10/1732639409437/Put/seqid=0 2024-11-26T16:43:33,285 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112643e52d1b3389497eaa91371947bfb352_c551e376aede51019d0df72805dd3c05, store=[table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:33,286 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112643e52d1b3389497eaa91371947bfb352_c551e376aede51019d0df72805dd3c05 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:33,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742427_1603 (size=12949) 2024-11-26T16:43:33,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742428_1604 (size=4469) 2024-11-26T16:43:33,291 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/3fa0ce86883b4a39943d88afd6671cfe as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/3fa0ce86883b4a39943d88afd6671cfe 2024-11-26T16:43:33,295 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c551e376aede51019d0df72805dd3c05/B of c551e376aede51019d0df72805dd3c05 into 3fa0ce86883b4a39943d88afd6671cfe(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:43:33,295 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:33,295 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., storeName=c551e376aede51019d0df72805dd3c05/B, priority=12, startTime=1732639413271; duration=0sec 2024-11-26T16:43:33,296 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-26T16:43:33,296 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c551e376aede51019d0df72805dd3c05:B 2024-11-26T16:43:33,296 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T16:43:33,296 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-26T16:43:33,297 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1541): c551e376aede51019d0df72805dd3c05/C is initiating minor compaction (all files) 2024-11-26T16:43:33,297 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c551e376aede51019d0df72805dd3c05/C in TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:33,297 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/1b78d9e1349b48d5a9e355bf5e16cd8e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/bbd14811740c49308a81855b7e3a8b01, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/437c436bf2304c4c9a9d13a531c8dfb1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/87defdab19934d7f838af475dd2ec4f3] into tmpdir=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp, totalSize=48.6 K 2024-11-26T16:43:33,297 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b78d9e1349b48d5a9e355bf5e16cd8e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732639406053 2024-11-26T16:43:33,297 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting bbd14811740c49308a81855b7e3a8b01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732639406146 2024-11-26T16:43:33,297 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 437c436bf2304c4c9a9d13a531c8dfb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732639408210 2024-11-26T16:43:33,298 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] compactions.Compactor(225): Compacting 87defdab19934d7f838af475dd2ec4f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732639409437 2024-11-26T16:43:33,305 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c551e376aede51019d0df72805dd3c05#C#compaction#509 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:43:33,305 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/3453b4e6d6bd4ef08a59ec7ecd744a01 is 50, key is test_row_0/C:col10/1732639409437/Put/seqid=0 2024-11-26T16:43:33,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742429_1605 (size=12949) 2024-11-26T16:43:33,690 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c551e376aede51019d0df72805dd3c05#A#compaction#507 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T16:43:33,691 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/2be6eaf1b2ad4b8691cd38567a425743 is 175, key is test_row_0/A:col10/1732639409437/Put/seqid=0 2024-11-26T16:43:33,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742430_1606 (size=31903) 2024-11-26T16:43:33,712 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/3453b4e6d6bd4ef08a59ec7ecd744a01 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/3453b4e6d6bd4ef08a59ec7ecd744a01 2024-11-26T16:43:33,715 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c551e376aede51019d0df72805dd3c05/C of c551e376aede51019d0df72805dd3c05 into 3453b4e6d6bd4ef08a59ec7ecd744a01(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:43:33,715 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:33,715 INFO [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., storeName=c551e376aede51019d0df72805dd3c05/C, priority=12, startTime=1732639413271; duration=0sec 2024-11-26T16:43:33,715 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:33,715 DEBUG [RS:0;2dedfe7466e8:38109-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c551e376aede51019d0df72805dd3c05:C 2024-11-26T16:43:34,098 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/2be6eaf1b2ad4b8691cd38567a425743 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/2be6eaf1b2ad4b8691cd38567a425743 2024-11-26T16:43:34,102 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c551e376aede51019d0df72805dd3c05/A of c551e376aede51019d0df72805dd3c05 into 2be6eaf1b2ad4b8691cd38567a425743(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T16:43:34,102 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c551e376aede51019d0df72805dd3c05: 2024-11-26T16:43:34,102 INFO [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., storeName=c551e376aede51019d0df72805dd3c05/A, priority=12, startTime=1732639413271; duration=0sec 2024-11-26T16:43:34,102 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T16:43:34,102 DEBUG [RS:0;2dedfe7466e8:38109-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c551e376aede51019d0df72805dd3c05:A 2024-11-26T16:43:34,353 INFO [Thread-2352 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2352. 2024-11-26T16:43:34,353 DEBUG [Thread-2352 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:43:34,353 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:34,353 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:34,353 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:34,372 INFO [Thread-2356 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2356. 2024-11-26T16:43:34,372 DEBUG [Thread-2356 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-26T16:43:34,372 DEBUG [Thread-2356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:34,372 DEBUG [Thread-2356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:34,372 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:34,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-26T16:43:34,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-26T16:43:34,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 127 2024-11-26T16:43:34,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-26T16:43:34,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-26T16:43:34,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-26T16:43:34,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-26T16:43:34,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2861 2024-11-26T16:43:34,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2768 2024-11-26T16:43:34,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2820 2024-11-26T16:43:34,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2864 2024-11-26T16:43:34,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2766 2024-11-26T16:43:34,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-26T16:43:34,372 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T16:43:34,372 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T16:43:34,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:34,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:34,373 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:34,373 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-26T16:43:34,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-26T16:43:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T16:43:34,376 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639414376"}]},"ts":"1732639414376"} 2024-11-26T16:43:34,377 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-26T16:43:34,377 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-26T16:43:34,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-26T16:43:34,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c551e376aede51019d0df72805dd3c05, UNASSIGN}] 2024-11-26T16:43:34,380 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c551e376aede51019d0df72805dd3c05, UNASSIGN 2024-11-26T16:43:34,380 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=c551e376aede51019d0df72805dd3c05, regionState=CLOSING, regionLocation=2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:34,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c551e376aede51019d0df72805dd3c05, UNASSIGN because future has completed 2024-11-26T16:43:34,382 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T16:43:34,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134}] 2024-11-26T16:43:34,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T16:43:34,534 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(122): Close c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:34,534 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T16:43:34,534 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1722): Closing c551e376aede51019d0df72805dd3c05, disabling compactions & flushes 2024-11-26T16:43:34,534 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:34,534 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:34,534 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. after waiting 0 ms 2024-11-26T16:43:34,534 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:34,534 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(2902): Flushing c551e376aede51019d0df72805dd3c05 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-26T16:43:34,535 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=A 2024-11-26T16:43:34,535 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:34,535 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=B 2024-11-26T16:43:34,535 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:34,535 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c551e376aede51019d0df72805dd3c05, store=C 2024-11-26T16:43:34,535 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-26T16:43:34,540 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112671dbd458b6d54eebbc95f54840b62a35_c551e376aede51019d0df72805dd3c05 is 50, key is test_row_0/A:col10/1732639412422/Put/seqid=0 2024-11-26T16:43:34,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742431_1607 (size=12454) 2024-11-26T16:43:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T16:43:34,944 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T16:43:34,947 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112671dbd458b6d54eebbc95f54840b62a35_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112671dbd458b6d54eebbc95f54840b62a35_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:34,947 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/d7cee42dd2ef4dc896628c95284e3562, store: [table=TestAcidGuarantees family=A region=c551e376aede51019d0df72805dd3c05] 2024-11-26T16:43:34,948 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/d7cee42dd2ef4dc896628c95284e3562 is 175, key is test_row_0/A:col10/1732639412422/Put/seqid=0 2024-11-26T16:43:34,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742432_1608 (size=31255) 2024-11-26T16:43:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T16:43:35,352 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=444, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/d7cee42dd2ef4dc896628c95284e3562 2024-11-26T16:43:35,357 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/5f50c0c5881c4a739bac7efdd17bd337 is 50, key is test_row_0/B:col10/1732639412422/Put/seqid=0 2024-11-26T16:43:35,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742433_1609 (size=12301) 2024-11-26T16:43:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T16:43:35,761 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/5f50c0c5881c4a739bac7efdd17bd337 2024-11-26T16:43:35,766 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/df55d18565c746e6a8c53ceda75ee01b is 50, key is test_row_0/C:col10/1732639412422/Put/seqid=0 2024-11-26T16:43:35,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742434_1610 (size=12301) 2024-11-26T16:43:36,170 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/df55d18565c746e6a8c53ceda75ee01b 2024-11-26T16:43:36,173 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/A/d7cee42dd2ef4dc896628c95284e3562 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/d7cee42dd2ef4dc896628c95284e3562 2024-11-26T16:43:36,176 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/d7cee42dd2ef4dc896628c95284e3562, entries=150, sequenceid=444, filesize=30.5 K 2024-11-26T16:43:36,176 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/B/5f50c0c5881c4a739bac7efdd17bd337 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/5f50c0c5881c4a739bac7efdd17bd337 2024-11-26T16:43:36,179 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/5f50c0c5881c4a739bac7efdd17bd337, entries=150, sequenceid=444, filesize=12.0 K 2024-11-26T16:43:36,179 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/.tmp/C/df55d18565c746e6a8c53ceda75ee01b as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/df55d18565c746e6a8c53ceda75ee01b 2024-11-26T16:43:36,182 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/df55d18565c746e6a8c53ceda75ee01b, entries=150, sequenceid=444, filesize=12.0 K 2024-11-26T16:43:36,183 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c551e376aede51019d0df72805dd3c05 in 1648ms, sequenceid=444, compaction requested=false 2024-11-26T16:43:36,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/6e36f0320b644d24820f586e5dba8f40, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/c9c983f4b81d47668d8506091410d9f5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/4e00ae5a046b41c68082fbb18e99b967, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/7927dc17f3ad44268d45d25eeec99c80, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/b5da16b950d8460fab0f78de95dae99d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/b8db8196b91943eba44ccf0cf73914fc, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/ea87091fe38b4c479cfa27c56d01c979, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/c85437f1f7584ee79198fc2b6210da41, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/f34518f56643454fa232325c3a084722, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/5aaf4885ff4744968bd9da8265821ff4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/fa8a825b2c5e4d38bc5dbb52cfe7551e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/3aa52fb94c094aeda4f2f709d42ace4f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/35710bbeb6fe459f8dabc2b26c45cb6d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/dfb0757af1c349cf80777e5a21b7cc60, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/ead513f24897446bb9dd145744f94c41, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/e0d74de5c85f4a03b4d5d54ed780a22b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/700dc4d6cc124b47a7141f810cb1c4e0, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/3d68f90aae8341e8a1315577d4832496, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/3630acf95c1a4a22ae3ea0db28357a08] to archive 2024-11-26T16:43:36,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T16:43:36,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/6e36f0320b644d24820f586e5dba8f40 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/6e36f0320b644d24820f586e5dba8f40 2024-11-26T16:43:36,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/c9c983f4b81d47668d8506091410d9f5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/c9c983f4b81d47668d8506091410d9f5 2024-11-26T16:43:36,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/4e00ae5a046b41c68082fbb18e99b967 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/4e00ae5a046b41c68082fbb18e99b967 2024-11-26T16:43:36,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/7927dc17f3ad44268d45d25eeec99c80 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/7927dc17f3ad44268d45d25eeec99c80 2024-11-26T16:43:36,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/b5da16b950d8460fab0f78de95dae99d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/b5da16b950d8460fab0f78de95dae99d 2024-11-26T16:43:36,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/b8db8196b91943eba44ccf0cf73914fc to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/b8db8196b91943eba44ccf0cf73914fc 2024-11-26T16:43:36,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/ea87091fe38b4c479cfa27c56d01c979 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/ea87091fe38b4c479cfa27c56d01c979 2024-11-26T16:43:36,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/c85437f1f7584ee79198fc2b6210da41 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/c85437f1f7584ee79198fc2b6210da41 2024-11-26T16:43:36,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/f34518f56643454fa232325c3a084722 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/f34518f56643454fa232325c3a084722 2024-11-26T16:43:36,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/5aaf4885ff4744968bd9da8265821ff4 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/5aaf4885ff4744968bd9da8265821ff4 2024-11-26T16:43:36,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/fa8a825b2c5e4d38bc5dbb52cfe7551e to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/fa8a825b2c5e4d38bc5dbb52cfe7551e 2024-11-26T16:43:36,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/3aa52fb94c094aeda4f2f709d42ace4f to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/3aa52fb94c094aeda4f2f709d42ace4f 2024-11-26T16:43:36,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/35710bbeb6fe459f8dabc2b26c45cb6d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/35710bbeb6fe459f8dabc2b26c45cb6d 2024-11-26T16:43:36,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/dfb0757af1c349cf80777e5a21b7cc60 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/dfb0757af1c349cf80777e5a21b7cc60 2024-11-26T16:43:36,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/ead513f24897446bb9dd145744f94c41 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/ead513f24897446bb9dd145744f94c41 2024-11-26T16:43:36,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/e0d74de5c85f4a03b4d5d54ed780a22b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/e0d74de5c85f4a03b4d5d54ed780a22b 2024-11-26T16:43:36,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/700dc4d6cc124b47a7141f810cb1c4e0 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/700dc4d6cc124b47a7141f810cb1c4e0 2024-11-26T16:43:36,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/3d68f90aae8341e8a1315577d4832496 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/3d68f90aae8341e8a1315577d4832496 2024-11-26T16:43:36,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/3630acf95c1a4a22ae3ea0db28357a08 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/3630acf95c1a4a22ae3ea0db28357a08 2024-11-26T16:43:36,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/ae694fbec5e04991946f23fb4088d7f4, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/7a18a615d6df42b4a6929eafa4942a7c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/30b28182fd9343ee9284180d44d12f51, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/73b8c69f92a849368783b82e59ff58bb, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/f5accc85c88643b4ab2f6be6bec56a8b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/c26fd24209114532bfaa675cae280fcd, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/9b9069e69fab4a9d96e31c67c7b1498f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/17a6618bd227489fbcd9a5bcafcf1de0, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/d4b31e0c3da145de9c0236437193a7aa, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/8a53f2789e064e41868c23afd1ee3aff, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/1abebbaea2a74dfca937d929015e1130, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/5152b5535ea54a63838c516529ad5c21, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/e690038834cc4fe6b63bf2b78489d8e6, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/b9cbf75e6c0c428faba33d37aaac115d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/a3de5ffbb52f4eedb2dfa6a68ec09367, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/9dd5271ec6a54cb481068e15c04e3705, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/a0ca6d063d004e6985c9fe2aada0b3f5, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/cf03af87f080444b9cbbea94f00d4ee2, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/8538e3d4ab934db9b22751bc65b7d828] to archive 2024-11-26T16:43:36,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T16:43:36,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/ae694fbec5e04991946f23fb4088d7f4 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/ae694fbec5e04991946f23fb4088d7f4 2024-11-26T16:43:36,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/7a18a615d6df42b4a6929eafa4942a7c to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/7a18a615d6df42b4a6929eafa4942a7c 2024-11-26T16:43:36,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/30b28182fd9343ee9284180d44d12f51 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/30b28182fd9343ee9284180d44d12f51 2024-11-26T16:43:36,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/73b8c69f92a849368783b82e59ff58bb to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/73b8c69f92a849368783b82e59ff58bb 2024-11-26T16:43:36,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/f5accc85c88643b4ab2f6be6bec56a8b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/f5accc85c88643b4ab2f6be6bec56a8b 2024-11-26T16:43:36,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/c26fd24209114532bfaa675cae280fcd to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/c26fd24209114532bfaa675cae280fcd 2024-11-26T16:43:36,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/9b9069e69fab4a9d96e31c67c7b1498f to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/9b9069e69fab4a9d96e31c67c7b1498f 2024-11-26T16:43:36,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/17a6618bd227489fbcd9a5bcafcf1de0 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/17a6618bd227489fbcd9a5bcafcf1de0 2024-11-26T16:43:36,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/d4b31e0c3da145de9c0236437193a7aa to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/d4b31e0c3da145de9c0236437193a7aa 2024-11-26T16:43:36,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/8a53f2789e064e41868c23afd1ee3aff to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/8a53f2789e064e41868c23afd1ee3aff 2024-11-26T16:43:36,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/1abebbaea2a74dfca937d929015e1130 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/1abebbaea2a74dfca937d929015e1130 2024-11-26T16:43:36,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/5152b5535ea54a63838c516529ad5c21 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/5152b5535ea54a63838c516529ad5c21 2024-11-26T16:43:36,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/e690038834cc4fe6b63bf2b78489d8e6 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/e690038834cc4fe6b63bf2b78489d8e6 2024-11-26T16:43:36,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/b9cbf75e6c0c428faba33d37aaac115d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/b9cbf75e6c0c428faba33d37aaac115d 2024-11-26T16:43:36,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/a3de5ffbb52f4eedb2dfa6a68ec09367 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/a3de5ffbb52f4eedb2dfa6a68ec09367 2024-11-26T16:43:36,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/9dd5271ec6a54cb481068e15c04e3705 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/9dd5271ec6a54cb481068e15c04e3705 2024-11-26T16:43:36,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/a0ca6d063d004e6985c9fe2aada0b3f5 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/a0ca6d063d004e6985c9fe2aada0b3f5 2024-11-26T16:43:36,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/cf03af87f080444b9cbbea94f00d4ee2 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/cf03af87f080444b9cbbea94f00d4ee2 2024-11-26T16:43:36,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/8538e3d4ab934db9b22751bc65b7d828 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/8538e3d4ab934db9b22751bc65b7d828 2024-11-26T16:43:36,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/f5f112ddfe974376ba0b782d7f224a26, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/0522e8a2c6294d84a0cd3dc5d1e401e2, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/4497716c9b9e47dc95598aaa1e105ac9, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/eb867314c5d34a87a49ebbd599774dc1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/e8d8c03626be40e49f5981b1c861ddc1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/2c6a4a3a97ca4992a4c348de82a7f92b, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/596328aaad9d4e4fbd3cfaa585be273d, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/9450d6f0adf64efeac2f6e929303e425, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/d162e1bb84534cc88ba44aaeb12341eb, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/30e4c913a7aa4700a339c36d9e0e3fbe, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/f183d2ff80df451ab0ee0b9212c99f75, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/2297e39bf135446b974641c724f132d9, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/109bb75a167045b2b11648ff112b2e8c, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/e4cabe134c444ed8a527041514ad9e2f, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/1b78d9e1349b48d5a9e355bf5e16cd8e, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/55916c67b2a74c1e934fd9ce933f5825, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/bbd14811740c49308a81855b7e3a8b01, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/437c436bf2304c4c9a9d13a531c8dfb1, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/87defdab19934d7f838af475dd2ec4f3] to archive 2024-11-26T16:43:36,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T16:43:36,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/f5f112ddfe974376ba0b782d7f224a26 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/f5f112ddfe974376ba0b782d7f224a26 2024-11-26T16:43:36,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/0522e8a2c6294d84a0cd3dc5d1e401e2 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/0522e8a2c6294d84a0cd3dc5d1e401e2 2024-11-26T16:43:36,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/4497716c9b9e47dc95598aaa1e105ac9 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/4497716c9b9e47dc95598aaa1e105ac9 2024-11-26T16:43:36,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/eb867314c5d34a87a49ebbd599774dc1 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/eb867314c5d34a87a49ebbd599774dc1 2024-11-26T16:43:36,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/e8d8c03626be40e49f5981b1c861ddc1 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/e8d8c03626be40e49f5981b1c861ddc1 2024-11-26T16:43:36,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/2c6a4a3a97ca4992a4c348de82a7f92b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/2c6a4a3a97ca4992a4c348de82a7f92b 2024-11-26T16:43:36,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/596328aaad9d4e4fbd3cfaa585be273d to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/596328aaad9d4e4fbd3cfaa585be273d 2024-11-26T16:43:36,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/9450d6f0adf64efeac2f6e929303e425 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/9450d6f0adf64efeac2f6e929303e425 2024-11-26T16:43:36,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/d162e1bb84534cc88ba44aaeb12341eb to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/d162e1bb84534cc88ba44aaeb12341eb 2024-11-26T16:43:36,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/30e4c913a7aa4700a339c36d9e0e3fbe to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/30e4c913a7aa4700a339c36d9e0e3fbe 2024-11-26T16:43:36,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/f183d2ff80df451ab0ee0b9212c99f75 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/f183d2ff80df451ab0ee0b9212c99f75 2024-11-26T16:43:36,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/2297e39bf135446b974641c724f132d9 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/2297e39bf135446b974641c724f132d9 2024-11-26T16:43:36,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/109bb75a167045b2b11648ff112b2e8c to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/109bb75a167045b2b11648ff112b2e8c 2024-11-26T16:43:36,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/e4cabe134c444ed8a527041514ad9e2f to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/e4cabe134c444ed8a527041514ad9e2f 2024-11-26T16:43:36,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/1b78d9e1349b48d5a9e355bf5e16cd8e to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/1b78d9e1349b48d5a9e355bf5e16cd8e 2024-11-26T16:43:36,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/55916c67b2a74c1e934fd9ce933f5825 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/55916c67b2a74c1e934fd9ce933f5825 2024-11-26T16:43:36,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/bbd14811740c49308a81855b7e3a8b01 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/bbd14811740c49308a81855b7e3a8b01 2024-11-26T16:43:36,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/437c436bf2304c4c9a9d13a531c8dfb1 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/437c436bf2304c4c9a9d13a531c8dfb1 2024-11-26T16:43:36,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/87defdab19934d7f838af475dd2ec4f3 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/87defdab19934d7f838af475dd2ec4f3 2024-11-26T16:43:36,237 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/recovered.edits/447.seqid, newMaxSeqId=447, maxSeqId=4 2024-11-26T16:43:36,238 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. 2024-11-26T16:43:36,238 DEBUG [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1676): Region close journal for c551e376aede51019d0df72805dd3c05: Waiting for close lock at 1732639414534Running coprocessor pre-close hooks at 1732639414534Disabling compacts and flushes for region at 1732639414534Disabling writes for close at 1732639414534Obtaining lock to block concurrent updates at 1732639414534Preparing flush snapshotting stores in c551e376aede51019d0df72805dd3c05 at 1732639414534Finished memstore snapshotting TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732639414535 (+1 ms)Flushing stores of TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05. at 1732639414535Flushing c551e376aede51019d0df72805dd3c05/A: creating writer at 1732639414536 (+1 ms)Flushing mob file c551e376aede51019d0df72805dd3c05/A: appending metadata at 1732639414540 (+4 ms)Flushing mob file c551e376aede51019d0df72805dd3c05/A: closing flushed file at 1732639414540Flushing c551e376aede51019d0df72805dd3c05/A: appending metadata at 1732639414947 (+407 ms)Flushing c551e376aede51019d0df72805dd3c05/A: closing flushed file at 1732639414947Flushing c551e376aede51019d0df72805dd3c05/B: creating writer at 1732639415355 (+408 ms)Flushing c551e376aede51019d0df72805dd3c05/B: appending metadata at 1732639415357 (+2 ms)Flushing c551e376aede51019d0df72805dd3c05/B: closing flushed file at 1732639415357Flushing c551e376aede51019d0df72805dd3c05/C: creating writer at 1732639415764 (+407 ms)Flushing c551e376aede51019d0df72805dd3c05/C: appending metadata at 1732639415766 (+2 ms)Flushing c551e376aede51019d0df72805dd3c05/C: closing flushed file at 1732639415766Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ed6a993: reopening flushed file at 1732639416173 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4756f2b3: reopening flushed file at 1732639416176 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3305e513: reopening flushed file at 1732639416179 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c551e376aede51019d0df72805dd3c05 in 1648ms, sequenceid=444, compaction requested=false at 1732639416183 (+4 ms)Writing region close event to WAL at 1732639416235 (+52 ms)Running coprocessor post-close hooks at 1732639416238 (+3 ms)Closed at 1732639416238 2024-11-26T16:43:36,240 INFO [RS_CLOSE_REGION-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(157): Closed c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:36,240 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=c551e376aede51019d0df72805dd3c05, regionState=CLOSED 2024-11-26T16:43:36,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 because future has completed 2024-11-26T16:43:36,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-26T16:43:36,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseRegionProcedure c551e376aede51019d0df72805dd3c05, server=2dedfe7466e8,38109,1732639238134 in 1.8600 sec 2024-11-26T16:43:36,245 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-11-26T16:43:36,245 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c551e376aede51019d0df72805dd3c05, UNASSIGN in 1.8640 sec 2024-11-26T16:43:36,246 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-26T16:43:36,246 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8670 sec 2024-11-26T16:43:36,247 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732639416247"}]},"ts":"1732639416247"} 2024-11-26T16:43:36,248 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-26T16:43:36,248 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-26T16:43:36,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8760 sec 2024-11-26T16:43:36,306 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T16:43:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-26T16:43:36,522 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:43:36,522 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-26T16:43:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:43:36,524 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:43:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T16:43:36,524 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=162, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:43:36,527 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:36,528 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A, FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B, FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C, FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/recovered.edits] 2024-11-26T16:43:36,530 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/2be6eaf1b2ad4b8691cd38567a425743 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/2be6eaf1b2ad4b8691cd38567a425743 2024-11-26T16:43:36,531 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/d7cee42dd2ef4dc896628c95284e3562 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/A/d7cee42dd2ef4dc896628c95284e3562 2024-11-26T16:43:36,533 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/3fa0ce86883b4a39943d88afd6671cfe to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/3fa0ce86883b4a39943d88afd6671cfe 2024-11-26T16:43:36,534 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/5f50c0c5881c4a739bac7efdd17bd337 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/B/5f50c0c5881c4a739bac7efdd17bd337 2024-11-26T16:43:36,535 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/3453b4e6d6bd4ef08a59ec7ecd744a01 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/3453b4e6d6bd4ef08a59ec7ecd744a01 2024-11-26T16:43:36,536 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/df55d18565c746e6a8c53ceda75ee01b to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/C/df55d18565c746e6a8c53ceda75ee01b 2024-11-26T16:43:36,538 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/recovered.edits/447.seqid to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05/recovered.edits/447.seqid 2024-11-26T16:43:36,539 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/default/TestAcidGuarantees/c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:36,539 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-26T16:43:36,539 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T16:43:36,540 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-26T16:43:36,542 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261c80ef9d13824c2b9e5a562f52964cae_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411261c80ef9d13824c2b9e5a562f52964cae_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:36,543 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112631f7181480264bfa94c579a2bc4d93cc_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112631f7181480264bfa94c579a2bc4d93cc_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:36,544 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112636709fd3f12d44b0b5c97ff92e6e5856_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112636709fd3f12d44b0b5c97ff92e6e5856_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:36,544 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126627231c7997e4cd6ba4613f06e588529_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126627231c7997e4cd6ba4613f06e588529_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:36,545 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126692cebc0bd174aa6bc319a0cc2d4b8a8_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126692cebc0bd174aa6bc319a0cc2d4b8a8_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:36,546 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112669afffe6cbbb4edaaa1c03d0572dae15_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112669afffe6cbbb4edaaa1c03d0572dae15_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:36,547 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112671127419875d41f499c94c7c30213f68_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112671127419875d41f499c94c7c30213f68_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:36,548 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112671dbd458b6d54eebbc95f54840b62a35_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112671dbd458b6d54eebbc95f54840b62a35_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:36,549 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267978922de13b48b9866b443d44e47a58_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267978922de13b48b9866b443d44e47a58_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:36,549 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267d9015810014429dbb73774dec980c6c_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411267d9015810014429dbb73774dec980c6c_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:36,550 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112689e7a4bcbfd1487e8983b7112e5590b7_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112689e7a4bcbfd1487e8983b7112e5590b7_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:36,551 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b46ba5688a68465b8231b90e10e4995c_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b46ba5688a68465b8231b90e10e4995c_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:36,552 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b6b9bab2ab2640049a8c56235e8be886_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126b6b9bab2ab2640049a8c56235e8be886_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:36,553 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cf8cee803a13437baafcb539a924d7b4_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126cf8cee803a13437baafcb539a924d7b4_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:36,554 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d67209aa590145a09a3034bf00b8b8d1_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126d67209aa590145a09a3034bf00b8b8d1_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:36,554 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e486f236cd584e4b92bf1af7422a0803_c551e376aede51019d0df72805dd3c05 to hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241126e486f236cd584e4b92bf1af7422a0803_c551e376aede51019d0df72805dd3c05 2024-11-26T16:43:36,555 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-26T16:43:36,557 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=162, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:43:36,558 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-26T16:43:36,560 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-26T16:43:36,561 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=162, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:43:36,561 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-26T16:43:36,561 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732639416561"}]},"ts":"9223372036854775807"} 2024-11-26T16:43:36,563 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-26T16:43:36,563 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c551e376aede51019d0df72805dd3c05, NAME => 'TestAcidGuarantees,,1732639386803.c551e376aede51019d0df72805dd3c05.', STARTKEY => '', ENDKEY => ''}] 2024-11-26T16:43:36,563 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-26T16:43:36,563 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732639416563"}]},"ts":"9223372036854775807"} 2024-11-26T16:43:36,564 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-26T16:43:36,565 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=162, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-26T16:43:36,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-11-26T16:43:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35233 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-26T16:43:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-26T16:43:36,632 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-26T16:43:36,641 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 253), OpenFileDescriptor=455 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=354 (was 414), ProcessCount=11 (was 11), AvailableMemoryMB=5107 (was 5133) 2024-11-26T16:43:36,641 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T16:43:36,641 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T16:43:36,641 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T16:43:36,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:36,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:36,641 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T16:43:36,641 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T16:43:36,641 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=611241949, stopped=false 2024-11-26T16:43:36,642 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2dedfe7466e8,35233,1732639237322 2024-11-26T16:43:36,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T16:43:36,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T16:43:36,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T16:43:36,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T16:43:36,644 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T16:43:36,644 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T16:43:36,644 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T16:43:36,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:36,645 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T16:43:36,645 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T16:43:36,645 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2dedfe7466e8,38109,1732639238134' ***** 2024-11-26T16:43:36,645 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T16:43:36,645 INFO [RS:0;2dedfe7466e8:38109 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T16:43:36,646 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T16:43:36,646 INFO [RS:0;2dedfe7466e8:38109 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T16:43:36,646 INFO [RS:0;2dedfe7466e8:38109 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T16:43:36,646 INFO [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(959): stopping server 2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:36,646 INFO [RS:0;2dedfe7466e8:38109 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T16:43:36,646 INFO [RS:0;2dedfe7466e8:38109 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2dedfe7466e8:38109. 2024-11-26T16:43:36,646 DEBUG [RS:0;2dedfe7466e8:38109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T16:43:36,646 DEBUG [RS:0;2dedfe7466e8:38109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:36,646 INFO [RS:0;2dedfe7466e8:38109 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T16:43:36,646 INFO [RS:0;2dedfe7466e8:38109 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T16:43:36,646 INFO [RS:0;2dedfe7466e8:38109 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T16:43:36,647 INFO [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T16:43:36,647 INFO [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T16:43:36,647 DEBUG [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-26T16:43:36,648 DEBUG [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T16:43:36,648 INFO [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T16:43:36,648 DEBUG [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T16:43:36,648 DEBUG [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T16:43:36,648 DEBUG [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T16:43:36,648 DEBUG [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T16:43:36,648 INFO [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-26T16:43:36,672 DEBUG [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/.tmp/info/3faa7a29e7464df6a300ddc29e74b8dd is 95, key is TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073./info:/1732639275261/DeleteFamily/seqid=0 2024-11-26T16:43:36,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742435_1611 (size=6514) 2024-11-26T16:43:36,708 INFO [regionserver/2dedfe7466e8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T16:43:36,848 DEBUG [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T16:43:37,048 DEBUG [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T16:43:37,076 INFO [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/.tmp/info/3faa7a29e7464df6a300ddc29e74b8dd 2024-11-26T16:43:37,094 DEBUG [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/.tmp/ns/d26e71c7b4194110816658446c6735c3 is 93, key is TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073./ns:/1732639275216/DeleteFamily/seqid=0 2024-11-26T16:43:37,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742436_1612 (size=5906) 2024-11-26T16:43:37,243 INFO [regionserver/2dedfe7466e8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T16:43:37,243 INFO [regionserver/2dedfe7466e8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T16:43:37,248 DEBUG [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T16:43:37,449 DEBUG [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T16:43:37,498 INFO [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/.tmp/ns/d26e71c7b4194110816658446c6735c3 2024-11-26T16:43:37,515 DEBUG [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/.tmp/rep_barrier/61fac5c8b1524d648cd03da5997aaaf1 is 102, key is TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073./rep_barrier:/1732639275216/DeleteFamily/seqid=0 2024-11-26T16:43:37,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742437_1613 (size=6025) 2024-11-26T16:43:37,649 DEBUG [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T16:43:37,781 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-26T16:43:37,849 DEBUG [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T16:43:37,919 INFO [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/.tmp/rep_barrier/61fac5c8b1524d648cd03da5997aaaf1 2024-11-26T16:43:37,937 DEBUG [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/.tmp/table/e06219b089a147809b7a5ba17d377d82 is 96, key is TestAcidGuarantees,,1732639241855.7c6dff940b7962766483b28d1008c073./table:/1732639275216/DeleteFamily/seqid=0 2024-11-26T16:43:37,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742438_1614 (size=5936) 2024-11-26T16:43:38,049 DEBUG [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T16:43:38,249 DEBUG [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T16:43:38,341 INFO [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/.tmp/table/e06219b089a147809b7a5ba17d377d82 2024-11-26T16:43:38,344 DEBUG [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/.tmp/info/3faa7a29e7464df6a300ddc29e74b8dd as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/info/3faa7a29e7464df6a300ddc29e74b8dd 2024-11-26T16:43:38,347 INFO [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/info/3faa7a29e7464df6a300ddc29e74b8dd, entries=12, sequenceid=90, filesize=6.4 K 2024-11-26T16:43:38,348 DEBUG [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/.tmp/ns/d26e71c7b4194110816658446c6735c3 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/ns/d26e71c7b4194110816658446c6735c3 2024-11-26T16:43:38,351 INFO [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/ns/d26e71c7b4194110816658446c6735c3, entries=8, sequenceid=90, filesize=5.8 K 2024-11-26T16:43:38,351 DEBUG [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/.tmp/rep_barrier/61fac5c8b1524d648cd03da5997aaaf1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/rep_barrier/61fac5c8b1524d648cd03da5997aaaf1 2024-11-26T16:43:38,354 INFO [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/rep_barrier/61fac5c8b1524d648cd03da5997aaaf1, entries=6, sequenceid=90, filesize=5.9 K 2024-11-26T16:43:38,354 DEBUG [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/.tmp/table/e06219b089a147809b7a5ba17d377d82 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/table/e06219b089a147809b7a5ba17d377d82 2024-11-26T16:43:38,357 INFO [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/table/e06219b089a147809b7a5ba17d377d82, entries=7, sequenceid=90, filesize=5.8 K 2024-11-26T16:43:38,357 INFO [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1709ms, sequenceid=90, compaction requested=false 2024-11-26T16:43:38,361 DEBUG [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-26T16:43:38,361 DEBUG [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T16:43:38,362 INFO [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T16:43:38,362 DEBUG [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732639416647Running coprocessor pre-close hooks at 1732639416648 (+1 ms)Disabling compacts and flushes for region at 1732639416648Disabling writes for close at 1732639416648Obtaining lock to block concurrent updates at 1732639416648Preparing flush snapshotting stores in 1588230740 at 1732639416648Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732639416649 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732639416649Flushing 1588230740/info: creating writer at 1732639416649Flushing 1588230740/info: appending metadata at 1732639416671 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732639416671Flushing 1588230740/ns: creating writer at 1732639417079 (+408 ms)Flushing 1588230740/ns: appending metadata at 1732639417094 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732639417094Flushing 1588230740/rep_barrier: creating writer at 1732639417501 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732639417515 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732639417515Flushing 1588230740/table: creating writer at 1732639417922 (+407 ms)Flushing 1588230740/table: appending metadata at 1732639417936 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732639417937 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@544c3b8: reopening flushed file at 1732639418344 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ac24bf9: reopening flushed file at 1732639418347 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19b7f37c: reopening flushed file at 1732639418351 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d1ea1f7: reopening flushed file at 1732639418354 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1709ms, sequenceid=90, compaction requested=false at 1732639418357 (+3 ms)Writing region close event to WAL at 1732639418359 (+2 ms)Running coprocessor post-close hooks at 1732639418361 (+2 ms)Closed at 1732639418361 2024-11-26T16:43:38,362 DEBUG [RS_CLOSE_META-regionserver/2dedfe7466e8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T16:43:38,450 INFO [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(976): stopping server 2dedfe7466e8,38109,1732639238134; all regions closed. 2024-11-26T16:43:38,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741834_1010 (size=25344) 2024-11-26T16:43:38,456 DEBUG [RS:0;2dedfe7466e8:38109 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/oldWALs 2024-11-26T16:43:38,457 INFO [RS:0;2dedfe7466e8:38109 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2dedfe7466e8%2C38109%2C1732639238134.meta:.meta(num 1732639241394) 2024-11-26T16:43:38,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741832_1008 (size=18479286) 2024-11-26T16:43:38,460 DEBUG [RS:0;2dedfe7466e8:38109 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/oldWALs 2024-11-26T16:43:38,460 INFO [RS:0;2dedfe7466e8:38109 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2dedfe7466e8%2C38109%2C1732639238134:(num 1732639240403) 2024-11-26T16:43:38,460 DEBUG [RS:0;2dedfe7466e8:38109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T16:43:38,460 INFO [RS:0;2dedfe7466e8:38109 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T16:43:38,460 INFO [RS:0;2dedfe7466e8:38109 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T16:43:38,460 INFO [RS:0;2dedfe7466e8:38109 {}] hbase.ChoreService(370): Chore service for: regionserver/2dedfe7466e8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T16:43:38,460 INFO [RS:0;2dedfe7466e8:38109 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T16:43:38,461 INFO [regionserver/2dedfe7466e8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T16:43:38,461 INFO [RS:0;2dedfe7466e8:38109 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38109 2024-11-26T16:43:38,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2dedfe7466e8,38109,1732639238134 2024-11-26T16:43:38,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T16:43:38,465 INFO [RS:0;2dedfe7466e8:38109 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T16:43:38,467 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2dedfe7466e8,38109,1732639238134] 2024-11-26T16:43:38,469 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2dedfe7466e8,38109,1732639238134 already deleted, retry=false 2024-11-26T16:43:38,469 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2dedfe7466e8,38109,1732639238134 expired; onlineServers=0 2024-11-26T16:43:38,469 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2dedfe7466e8,35233,1732639237322' ***** 2024-11-26T16:43:38,469 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T16:43:38,469 INFO [M:0;2dedfe7466e8:35233 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T16:43:38,469 INFO [M:0;2dedfe7466e8:35233 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T16:43:38,469 DEBUG [M:0;2dedfe7466e8:35233 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T16:43:38,469 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T16:43:38,470 DEBUG [M:0;2dedfe7466e8:35233 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T16:43:38,470 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster-HFileCleaner.large.0-1732639240130 {}] cleaner.HFileCleaner(306): Exit Thread[master/2dedfe7466e8:0:becomeActiveMaster-HFileCleaner.large.0-1732639240130,5,FailOnTimeoutGroup] 2024-11-26T16:43:38,470 DEBUG [master/2dedfe7466e8:0:becomeActiveMaster-HFileCleaner.small.0-1732639240131 {}] cleaner.HFileCleaner(306): Exit Thread[master/2dedfe7466e8:0:becomeActiveMaster-HFileCleaner.small.0-1732639240131,5,FailOnTimeoutGroup] 2024-11-26T16:43:38,470 INFO [M:0;2dedfe7466e8:35233 {}] hbase.ChoreService(370): Chore service for: master/2dedfe7466e8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T16:43:38,470 INFO [M:0;2dedfe7466e8:35233 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T16:43:38,470 DEBUG [M:0;2dedfe7466e8:35233 {}] master.HMaster(1795): Stopping service threads 2024-11-26T16:43:38,470 INFO [M:0;2dedfe7466e8:35233 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T16:43:38,470 INFO [M:0;2dedfe7466e8:35233 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T16:43:38,470 ERROR [M:0;2dedfe7466e8:35233 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (879580466) connection to localhost/127.0.0.1:44115 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:44115,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-26T16:43:38,471 INFO [M:0;2dedfe7466e8:35233 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T16:43:38,471 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T16:43:38,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T16:43:38,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T16:43:38,472 DEBUG [M:0;2dedfe7466e8:35233 {}] zookeeper.ZKUtil(347): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T16:43:38,472 WARN [M:0;2dedfe7466e8:35233 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T16:43:38,472 INFO [M:0;2dedfe7466e8:35233 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/.lastflushedseqids 2024-11-26T16:43:38,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742439_1615 (size=72) 2024-11-26T16:43:38,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T16:43:38,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38109-0x10034b45a0d0001, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T16:43:38,568 INFO [RS:0;2dedfe7466e8:38109 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T16:43:38,569 INFO [RS:0;2dedfe7466e8:38109 {}] regionserver.HRegionServer(1031): Exiting; stopping=2dedfe7466e8,38109,1732639238134; zookeeper connection closed. 2024-11-26T16:43:38,569 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3c401323 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3c401323 2024-11-26T16:43:38,569 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-26T16:43:38,882 INFO [M:0;2dedfe7466e8:35233 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T16:43:38,882 INFO [M:0;2dedfe7466e8:35233 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T16:43:38,883 DEBUG [M:0;2dedfe7466e8:35233 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T16:43:38,883 INFO [M:0;2dedfe7466e8:35233 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T16:43:38,883 DEBUG [M:0;2dedfe7466e8:35233 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T16:43:38,883 DEBUG [M:0;2dedfe7466e8:35233 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T16:43:38,883 DEBUG [M:0;2dedfe7466e8:35233 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T16:43:38,883 INFO [M:0;2dedfe7466e8:35233 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=750.38 KB heapSize=917.20 KB 2024-11-26T16:43:38,898 DEBUG [M:0;2dedfe7466e8:35233 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fb8da42a894745059f8bb6c5fe1d57eb is 82, key is hbase:meta,,1/info:regioninfo/1732639241479/Put/seqid=0 2024-11-26T16:43:38,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742440_1616 (size=5672) 2024-11-26T16:43:39,302 INFO [M:0;2dedfe7466e8:35233 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2055 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fb8da42a894745059f8bb6c5fe1d57eb 2024-11-26T16:43:39,331 DEBUG [M:0;2dedfe7466e8:35233 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/33d53bd4328c400c8b685ff1fcefb798 is 2480, key is \x00\x00\x00\x00\x00\x00\x00!/proc:d/1732639278474/Put/seqid=0 2024-11-26T16:43:39,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742441_1617 (size=44030) 2024-11-26T16:43:39,735 INFO [M:0;2dedfe7466e8:35233 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=749.82 KB at sequenceid=2055 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/33d53bd4328c400c8b685ff1fcefb798 2024-11-26T16:43:39,739 INFO [M:0;2dedfe7466e8:35233 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 33d53bd4328c400c8b685ff1fcefb798 2024-11-26T16:43:39,755 DEBUG [M:0;2dedfe7466e8:35233 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9374d696467a4ed0892d872604a6c8e1 is 69, key is 2dedfe7466e8,38109,1732639238134/rs:state/1732639240161/Put/seqid=0 2024-11-26T16:43:39,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073742442_1618 (size=5156) 2024-11-26T16:43:40,158 INFO [M:0;2dedfe7466e8:35233 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2055 (bloomFilter=true), to=hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9374d696467a4ed0892d872604a6c8e1 2024-11-26T16:43:40,162 DEBUG [M:0;2dedfe7466e8:35233 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fb8da42a894745059f8bb6c5fe1d57eb as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fb8da42a894745059f8bb6c5fe1d57eb 2024-11-26T16:43:40,166 INFO [M:0;2dedfe7466e8:35233 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fb8da42a894745059f8bb6c5fe1d57eb, entries=8, sequenceid=2055, filesize=5.5 K 2024-11-26T16:43:40,166 DEBUG [M:0;2dedfe7466e8:35233 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/33d53bd4328c400c8b685ff1fcefb798 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/33d53bd4328c400c8b685ff1fcefb798 2024-11-26T16:43:40,169 INFO [M:0;2dedfe7466e8:35233 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 33d53bd4328c400c8b685ff1fcefb798 2024-11-26T16:43:40,169 INFO [M:0;2dedfe7466e8:35233 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/33d53bd4328c400c8b685ff1fcefb798, entries=162, sequenceid=2055, filesize=43.0 K 2024-11-26T16:43:40,170 DEBUG [M:0;2dedfe7466e8:35233 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9374d696467a4ed0892d872604a6c8e1 as hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9374d696467a4ed0892d872604a6c8e1 2024-11-26T16:43:40,172 INFO [M:0;2dedfe7466e8:35233 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44115/user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9374d696467a4ed0892d872604a6c8e1, entries=1, sequenceid=2055, filesize=5.0 K 2024-11-26T16:43:40,173 INFO [M:0;2dedfe7466e8:35233 {}] regionserver.HRegion(3140): Finished flush of dataSize ~750.38 KB/768389, heapSize ~916.90 KB/938904, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1290ms, sequenceid=2055, compaction requested=false 2024-11-26T16:43:40,174 INFO [M:0;2dedfe7466e8:35233 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T16:43:40,174 DEBUG [M:0;2dedfe7466e8:35233 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732639418883Disabling compacts and flushes for region at 1732639418883Disabling writes for close at 1732639418883Obtaining lock to block concurrent updates at 1732639418883Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732639418883Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=768389, getHeapSize=939144, getOffHeapSize=0, getCellsCount=2242 at 1732639418883Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732639418884 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732639418884Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732639418898 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732639418898Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732639419306 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732639419331 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732639419331Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732639419739 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732639419754 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732639419754Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c088ea6: reopening flushed file at 1732639420162 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19c9cc84: reopening flushed file at 1732639420166 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47ded1fc: reopening flushed file at 1732639420169 (+3 ms)Finished flush of dataSize ~750.38 KB/768389, heapSize ~916.90 KB/938904, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1290ms, sequenceid=2055, compaction requested=false at 1732639420173 (+4 ms)Writing region close event to WAL at 1732639420174 (+1 ms)Closed at 1732639420174 2024-11-26T16:43:40,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41979 is added to blk_1073741830_1006 (size=903131) 2024-11-26T16:43:40,176 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/746c1ac5-f1f2-fdef-feb1-60f32eb50711/MasterData/WALs/2dedfe7466e8,35233,1732639237322/2dedfe7466e8%2C35233%2C1732639237322.1732639239479 not finished, retry = 0 2024-11-26T16:43:40,277 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T16:43:40,278 INFO [M:0;2dedfe7466e8:35233 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T16:43:40,278 INFO [M:0;2dedfe7466e8:35233 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35233 2024-11-26T16:43:40,278 INFO [M:0;2dedfe7466e8:35233 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T16:43:40,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T16:43:40,380 INFO [M:0;2dedfe7466e8:35233 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T16:43:40,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35233-0x10034b45a0d0000, quorum=127.0.0.1:49596, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T16:43:40,385 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@292d50c4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T16:43:40,387 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T16:43:40,387 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T16:43:40,387 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T16:43:40,388 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/hadoop.log.dir/,STOPPED} 2024-11-26T16:43:40,390 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T16:43:40,390 WARN [BP-338185642-172.17.0.2-1732639233957 heartbeating to localhost/127.0.0.1:44115 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T16:43:40,391 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T16:43:40,391 WARN [BP-338185642-172.17.0.2-1732639233957 heartbeating to localhost/127.0.0.1:44115 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-338185642-172.17.0.2-1732639233957 (Datanode Uuid 9ade134c-bb6c-4bdc-9e79-03c00e4a8431) service to localhost/127.0.0.1:44115 2024-11-26T16:43:40,393 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/cluster_5c35ce1e-289c-85d9-824c-12c667b295d2/data/data1/current/BP-338185642-172.17.0.2-1732639233957 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T16:43:40,393 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/cluster_5c35ce1e-289c-85d9-824c-12c667b295d2/data/data2/current/BP-338185642-172.17.0.2-1732639233957 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T16:43:40,394 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T16:43:40,405 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2341a969{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T16:43:40,406 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T16:43:40,406 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T16:43:40,406 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T16:43:40,406 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/22f0b568-5073-2478-15ff-6faf84a9c9ed/hadoop.log.dir/,STOPPED} 2024-11-26T16:43:40,421 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T16:43:40,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down